From 02dde83603342a7edcfc5399c2dbf1fe7fbf34b5 Mon Sep 17 00:00:00 2001 From: lgomezm Date: Sun, 17 Apr 2022 10:28:51 -0500 Subject: [PATCH 01/18] Started migration --- .../source-freshdesk/source_freshdesk/api.py | 2 +- .../source_freshdesk/client.py | 1 + .../source_freshdesk/source.py | 39 ++++- .../source_freshdesk/streams.py | 137 ++++++++++++++++++ 4 files changed, 174 insertions(+), 5 deletions(-) create mode 100644 airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/api.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/api.py index a9e0021d2010c..603d0263ab67a 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/api.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/api.py @@ -340,7 +340,7 @@ class ConversationsAPI(ClientIncrementalStreamAPI): def list(self, fields: Sequence[str] = None) -> Iterator[dict]: """Iterate over entities""" - tickets = TicketsAPI(self._api) + tickets = TicketsAPI(self._api) if self.state: tickets.state = self.state for ticket in tickets.list(): diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/client.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/client.py index 325088ad7f2c0..9d928337d09f0 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/client.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/client.py @@ -3,6 +3,7 @@ # +from airbyte_cdk.entrypoint import logger from typing import Any, Iterable, Mapping, Tuple from airbyte_cdk.models import AirbyteStream diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py index aa5427887ef17..a0f1ece62d4d3 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py @@ -2,11 +2,42 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # +import copy +import logging +from typing import Any, Iterator, List, Mapping, MutableMapping, Optional, Tuple -from airbyte_cdk.sources.deprecated.base_source import BaseSource +from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.deprecated.base_source import ConfiguredAirbyteStream +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.utils.schema_helpers import InternalConfig, split_config +from airbyte_cdk.utils.event_timing import create_timer +from requests import HTTPError +from source_freshdesk.api import API +from source_freshdesk.errors import FreshdeskError, FreshdeskNotFound, FreshdeskUnauthorized +from source_freshdesk.streams import Agents, Contacts +from requests.auth import HTTPBasicAuth -from .client import Client +class SourceFreshdesk(AbstractSource): -class SourceFreshdesk(BaseSource): - client_class = Client + def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: + alive = True + error_msg = None + try: + api = API(domain=config['domain'], api_key=config['api_key']) + api.get("settings/helpdesk") + except (FreshdeskUnauthorized, FreshdeskNotFound): + alive = False + error_msg = "Invalid credentials" + except FreshdeskError as error: + alive = False + error_msg = repr(error) + + return alive, error_msg + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + return [ + Agents(authenticator=HTTPBasicAuth(username=config["api_key"], password="X"), config=config), + Contacts(authenticator=HTTPBasicAuth(username=config["api_key"], password="X"), config=config), + ] diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py new file mode 100644 index 0000000000000..efeef441618dc --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py @@ -0,0 +1,137 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional +from urllib import parse + +import pendulum +import requests +import re +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.http import HttpStream +from requests.auth import AuthBase +from source_freshdesk.utils import CallCredit + + +LINK_REGEX = re.compile(r'<(.*?)>;\s*rel="next"') + + +class FreshdeskStream(HttpStream, ABC): + """Basic stream API that allows to iterate over entities""" + call_credit = 1 # see https://developers.freshdesk.com/api/#embedding + + def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], *args, **kwargs): + super().__init__(authenticator=authenticator) + requests_per_minute = config["requests_per_minute"] + start_date = config["start_date"] + self.domain = config["domain"] + self._call_credit = CallCredit(balance=requests_per_minute) if requests_per_minute else None + # By default, only tickets that have been created within the past 30 days will be returned. + # Since this logic rely not on updated tickets, it can break tickets dependant streams - conversations. + # So updated_since parameter will be always used in tickets streams. And start_date will be used too + # with default value 30 days look back. + self.start_date = pendulum.parse(start_date) if start_date else pendulum.now() - pendulum.duration(days=30) + + @property + def url_base(self) -> str: + return f"https://{self.domain.rstrip('/')}/api/v2" + + def backoff_time(self, response: requests.Response) -> Optional[float]: + if response.status_code == requests.codes.too_many_requests: + return float(response.headers.get("Retry-After")) + + def request_headers( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> Mapping[str, Any]: + return { + "Content-Type": "application/json", + "User-Agent": "Airbyte", + } + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + try: + link_header = response.headers.get("Link") + if not link_header: + return {} + match = LINK_REGEX.search(link_header) + next_url = match.group(1) + params = parse.parse_qs(parse.urlparse(next_url).query) + return {"per_page": params['per_page'][0], "page": params['page'][0]} + except Exception as e: + raise KeyError(f"error parsing next_page token: {e}") + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + params = {"per_page": 100} + if next_page_token and "page" in next_page_token: + params["page"] = next_page_token["page"] + return params + + def _consume_credit(self, credit): + """Consume call credit, if there is no credit left within current window will sleep til next period""" + if self._call_credit: + self._call_credit.consume(credit) + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Mapping[str, Any]]: + self._consume_credit(self.call_credit) + yield from super().read_records(sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state) + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + data = response.json() + if not data: + return + for element in data: + yield element + + +class IncrementalFreshdeskStream(FreshdeskStream, ABC): + state_filter = "updated_since" + + @property + def cursor_field(self) -> str: + return "updated_at" + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + current_stream_state = current_stream_state or {} + + current_stream_state_date = current_stream_state.get("updated_at", self.start_date) + if isinstance(current_stream_state_date, str): + current_stream_state_date = pendulum.parse(current_stream_state_date) + latest_record_date = pendulum.parse(latest_record.get("updated_at")) if latest_record.get("updated_at") else self.start_date + + return {"updated_at": max(current_stream_state_date, latest_record_date)} + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + if "updated_at" in stream_state: + params[self.state_filter] = stream_state["updated_at"] + else: + params[self.state_filter] = self.start_date + return params + + +class Agents(FreshdeskStream): + primary_key = "id" + + def path(self, **kwargs) -> str: + return "agents" + + +class Contacts(IncrementalFreshdeskStream): + state_filter = "_updated_since" + primary_key = "id" + + def path(self, **kwargs) -> str: + return "contacts" \ No newline at end of file From 45148e226f2ce5a5c30e5ea52ff43fe6b97ce0f5 Mon Sep 17 00:00:00 2001 From: lgomezm Date: Sun, 17 Apr 2022 10:38:50 -0500 Subject: [PATCH 02/18] Migrated more streams --- .../source_freshdesk/streams.py | 42 +++++++++++++++++-- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py index efeef441618dc..40aec31b26a6b 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py @@ -22,6 +22,7 @@ class FreshdeskStream(HttpStream, ABC): """Basic stream API that allows to iterate over entities""" call_credit = 1 # see https://developers.freshdesk.com/api/#embedding + primary_key = "id" def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], *args, **kwargs): super().__init__(authenticator=authenticator) @@ -123,15 +124,50 @@ def request_params( class Agents(FreshdeskStream): - primary_key = "id" def path(self, **kwargs) -> str: return "agents" +class Companies(FreshdeskStream): + + def path(self, **kwargs) -> str: + return "companies" + + class Contacts(IncrementalFreshdeskStream): state_filter = "_updated_since" - primary_key = "id" def path(self, **kwargs) -> str: - return "contacts" \ No newline at end of file + return "contacts" + + +class Groups(FreshdeskStream): + + def path(self, **kwargs) -> str: + return "groups" + + +class Roles(FreshdeskStream): + + def path(self, **kwargs) -> str: + return "roles" + + +class Skills(FreshdeskStream): + + def path(self, **kwargs) -> str: + return "skills" + + +class TimeEntries(FreshdeskStream): + + def path(self, **kwargs) -> str: + return "time_entries" + + +class SatisfactionRatings(IncrementalFreshdeskStream): + state_filter = "created_since" + + def path(self, **kwargs) -> str: + return "surveys/satisfaction_ratings" From 5b4113797e7fcf937dfad8b94a76aa63f6b74278 Mon Sep 17 00:00:00 2001 From: lgomezm Date: Sun, 17 Apr 2022 10:55:49 -0500 Subject: [PATCH 03/18] Refactored usage of authenticator --- .../source-freshdesk/source_freshdesk/source.py | 13 +++++++++---- .../source-freshdesk/source_freshdesk/streams.py | 6 +++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py index a0f1ece62d4d3..844746f2aa9c6 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py @@ -15,8 +15,7 @@ from requests import HTTPError from source_freshdesk.api import API from source_freshdesk.errors import FreshdeskError, FreshdeskNotFound, FreshdeskUnauthorized -from source_freshdesk.streams import Agents, Contacts -from requests.auth import HTTPBasicAuth +from source_freshdesk.streams import Agents, Companies, Contacts, Groups, Roles, SatisfactionRatings, Skills, TimeEntries class SourceFreshdesk(AbstractSource): @@ -38,6 +37,12 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> def streams(self, config: Mapping[str, Any]) -> List[Stream]: return [ - Agents(authenticator=HTTPBasicAuth(username=config["api_key"], password="X"), config=config), - Contacts(authenticator=HTTPBasicAuth(username=config["api_key"], password="X"), config=config), + Agents(config=config), + Companies(config=config), + Contacts(config=config), + Groups(config=config), + Roles(config=config), + Skills(config=config), + TimeEntries(config=config), + SatisfactionRatings(config=config) ] diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py index 40aec31b26a6b..0358ccf12a22b 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py @@ -12,7 +12,7 @@ import re from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream -from requests.auth import AuthBase +from requests.auth import HTTPBasicAuth from source_freshdesk.utils import CallCredit @@ -24,8 +24,8 @@ class FreshdeskStream(HttpStream, ABC): call_credit = 1 # see https://developers.freshdesk.com/api/#embedding primary_key = "id" - def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], *args, **kwargs): - super().__init__(authenticator=authenticator) + def __init__(self, config: Mapping[str, Any], *args, **kwargs): + super().__init__(authenticator=HTTPBasicAuth(username=config["api_key"], password="unused_with_api_key")) requests_per_minute = config["requests_per_minute"] start_date = config["start_date"] self.domain = config["domain"] From 9b8f1523bddd633cfd50b4ac765330361c322081 Mon Sep 17 00:00:00 2001 From: lgomezm Date: Sun, 17 Apr 2022 15:23:08 -0500 Subject: [PATCH 04/18] Migrated tickets and conversations --- .../source_freshdesk/source.py | 52 +++++----- .../source_freshdesk/streams.py | 96 ++++++++++++++++++- 2 files changed, 120 insertions(+), 28 deletions(-) diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py index 844746f2aa9c6..9638546d8d3ad 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py @@ -2,47 +2,51 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # -import copy import logging -from typing import Any, Iterator, List, Mapping, MutableMapping, Optional, Tuple +import requests +from typing import Any, List, Mapping, Optional, Tuple +from requests.auth import AuthBase, HTTPBasicAuth -from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.deprecated.base_source import ConfiguredAirbyteStream from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.utils.schema_helpers import InternalConfig, split_config -from airbyte_cdk.utils.event_timing import create_timer -from requests import HTTPError -from source_freshdesk.api import API -from source_freshdesk.errors import FreshdeskError, FreshdeskNotFound, FreshdeskUnauthorized -from source_freshdesk.streams import Agents, Companies, Contacts, Groups, Roles, SatisfactionRatings, Skills, TimeEntries +from source_freshdesk.streams import Agents, Companies, Contacts, Conversations, Groups, Roles, SatisfactionRatings, Skills, Tickets, TimeEntries class SourceFreshdesk(AbstractSource): + def _create_authenticator(self, api_key: str) -> AuthBase: + return HTTPBasicAuth(username=api_key, password="unused_with_api_key") + def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: alive = True error_msg = None try: - api = API(domain=config['domain'], api_key=config['api_key']) - api.get("settings/helpdesk") - except (FreshdeskUnauthorized, FreshdeskNotFound): - alive = False - error_msg = "Invalid credentials" - except FreshdeskError as error: + url = f"https://{config['domain'].rstrip('/')}/api/v2/settings/helpdesk" + r = requests.get(url=url, auth=self._create_authenticator(config["api_key"])) + if not r.ok: + alive = False + try: + body = r.json() + error_msg = f"{body.get('code')}: {body['message']}" + except ValueError: + error_msg = "Invalid credentials" + except Exception as error: alive = False error_msg = repr(error) return alive, error_msg def streams(self, config: Mapping[str, Any]) -> List[Stream]: + authenticator = self._create_authenticator(config["api_key"]) return [ - Agents(config=config), - Companies(config=config), - Contacts(config=config), - Groups(config=config), - Roles(config=config), - Skills(config=config), - TimeEntries(config=config), - SatisfactionRatings(config=config) + Agents(authenticator=authenticator, config=config), + Companies(authenticator=authenticator, config=config), + Contacts(authenticator=authenticator, config=config), + Conversations(authenticator=authenticator, config=config), + Groups(authenticator=authenticator, config=config), + Roles(authenticator=authenticator, config=config), + Skills(authenticator=authenticator, config=config), + TimeEntries(authenticator=authenticator, config=config), + Tickets(authenticator=authenticator, config=config), + SatisfactionRatings(authenticator=authenticator, config=config) ] diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py index 0358ccf12a22b..41af7904f4585 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py @@ -10,9 +10,11 @@ import pendulum import requests import re +from airbyte_cdk.entrypoint import logger # FIXME (Eugene K): use standard logger from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream -from requests.auth import HTTPBasicAuth +from airbyte_cdk.sources.utils.sentry import AirbyteSentry +from requests.auth import AuthBase from source_freshdesk.utils import CallCredit @@ -24,8 +26,8 @@ class FreshdeskStream(HttpStream, ABC): call_credit = 1 # see https://developers.freshdesk.com/api/#embedding primary_key = "id" - def __init__(self, config: Mapping[str, Any], *args, **kwargs): - super().__init__(authenticator=HTTPBasicAuth(username=config["api_key"], password="unused_with_api_key")) + def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], *args, **kwargs): + super().__init__(authenticator=authenticator) requests_per_minute = config["requests_per_minute"] start_date = config["start_date"] self.domain = config["domain"] @@ -96,7 +98,6 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp class IncrementalFreshdeskStream(FreshdeskStream, ABC): - state_filter = "updated_since" @property def cursor_field(self) -> str: @@ -166,6 +167,93 @@ def path(self, **kwargs) -> str: return "time_entries" +class Tickets(IncrementalFreshdeskStream): + state_filter = "updated_since" + ticket_paginate_limit = 300 + + def path(self, **kwargs) -> str: + return "tickets" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + if next_page_token and "updated_since" in next_page_token: + params["updated_since"] = next_page_token["updated_since"] + return params + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Mapping[str, Any]]: + """ + Read ticket records + + This block extends Incremental stream to overcome '300 page' server error. + Since the Ticket endpoint has a 300 page pagination limit, after 300 pages, update the parameters with + query using 'updated_since' = last_record, if there is more data remaining. + """ + stream_state = stream_state or {} + pagination_complete = False + + next_page_token = None + page = 1 + with AirbyteSentry.start_transaction("read_records", self.name), AirbyteSentry.start_transaction_span("read_records"): + while not pagination_complete: + request_headers = self.request_headers( + stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token + ) + request = self._create_prepared_request( + path=self.path(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + headers=dict(request_headers, **self.authenticator.get_auth_header()), + params=self.request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + json=self.request_body_json(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + data=self.request_body_data(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + ) + response = self._send_request(request, {}) + tickets = response.json() + yield from tickets + + next_page_token = self.next_page_token(response) + # checkpoint & switch the pagination + if page == self.ticket_paginate_limit and next_page_token: + # get last_record from latest batch, pos. -1, because of ACS order of records + last_record_updated_at = tickets[-1]["updated_at"] + page = 0 # reset page counter + last_record_updated_at = pendulum.parse(last_record_updated_at) + # updating request parameters with last_record state + next_page_token["updated_since"] = last_record_updated_at + # Increment page + page += 1 + + if not next_page_token: + pagination_complete = True + + # Always return an empty generator just in case no records were ever yielded + yield from [] + + +class Conversations(FreshdeskStream): + """Notes and Replies""" + def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], *args, **kwargs): + super().__init__(**kwargs) + self.tickets_stream = Tickets(authenticator=authenticator, config=config) + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"tickets/{stream_slice['id']}/conversations" + + def stream_slices( + self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + for ticket in self.tickets_stream.read_records( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice={}, stream_state={} + ): + yield {'id': ticket['id']} + + class SatisfactionRatings(IncrementalFreshdeskStream): state_filter = "created_since" From 4922269b9b65b9c998e7dfb64bb417194aad055d Mon Sep 17 00:00:00 2001 From: lgomezm Date: Mon, 18 Apr 2022 12:43:02 -0500 Subject: [PATCH 05/18] Code cleaned up --- .../source-freshdesk/source_freshdesk/api.py | 358 ------------------ .../source_freshdesk/client.py | 89 ----- .../source_freshdesk/errors.py | 37 -- .../source_freshdesk/streams.py | 2 +- .../source_freshdesk/utils.py | 50 --- 5 files changed, 1 insertion(+), 535 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-freshdesk/source_freshdesk/api.py delete mode 100644 airbyte-integrations/connectors/source-freshdesk/source_freshdesk/client.py delete mode 100644 airbyte-integrations/connectors/source-freshdesk/source_freshdesk/errors.py diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/api.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/api.py deleted file mode 100644 index 603d0263ab67a..0000000000000 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/api.py +++ /dev/null @@ -1,358 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# - - -from abc import ABC, abstractmethod -from functools import partial -from itertools import count -from typing import Any, Callable, Iterator, Mapping, MutableMapping, Optional, Sequence - -import pendulum -import requests -from airbyte_cdk.entrypoint import logger # FIXME (Eugene K): use standard logger -from requests import HTTPError -from source_freshdesk.errors import ( - FreshdeskAccessDenied, - FreshdeskBadRequest, - FreshdeskError, - FreshdeskNotFound, - FreshdeskRateLimited, - FreshdeskServerError, - FreshdeskUnauthorized, -) -from source_freshdesk.utils import CallCredit, retry_after_handler, retry_connection_handler - - -class API: - def __init__( - self, - domain: str, - api_key: str, - requests_per_minute: int = None, - verify: bool = True, - proxies: MutableMapping[str, Any] = None, - start_date: str = None, - ): - """Basic HTTP interface to read from endpoints""" - self._api_prefix = f"https://{domain.rstrip('/')}/api/v2/" - self._session = requests.Session() - self._session.auth = (api_key, "unused_with_api_key") - self._session.verify = verify - self._session.proxies = proxies - self._session.headers = { - "Content-Type": "application/json", - "User-Agent": "Airbyte", - } - - self._call_credit = CallCredit(balance=requests_per_minute) if requests_per_minute else None - - # By default, only tickets that have been created within the past 30 days will be returned. - # Since this logic rely not on updated tickets, it can break tickets dependant streams - conversations. - # So updated_since parameter will be always used in tickets streams. And start_date will be used too - # with default value 30 days look back. - self._start_date = pendulum.parse(start_date) if start_date else pendulum.now() - pendulum.duration(days=30) - - if domain.find("freshdesk.com") < 0: - raise AttributeError("Freshdesk v2 API works only via Freshdesk domains and not via custom CNAMEs") - - @staticmethod - def _parse_and_handle_errors(response): - try: - body = response.json() - except ValueError: - body = {} - - error_message = "Freshdesk Request Failed" - if "errors" in body: - error_message = f"{body.get('description')}: {body['errors']}" - # API docs don't mention this clearly, but in the case of bad credentials the returned JSON will have a - # "message" field at the top level - elif "message" in body: - error_message = f"{body.get('code')}: {body['message']}" - - if response.status_code == 400: - raise FreshdeskBadRequest(error_message or "Wrong input, check your data", response=response) - elif response.status_code == 401: - raise FreshdeskUnauthorized(error_message or "Invalid credentials", response=response) - elif response.status_code == 403: - raise FreshdeskAccessDenied(error_message or "You don't have enough permissions", response=response) - elif response.status_code == 404: - raise FreshdeskNotFound(error_message or "Resource not found", response=response) - elif response.status_code == 429: - retry_after = response.headers.get("Retry-After") - raise FreshdeskRateLimited( - f"429 Rate Limit Exceeded: API rate-limit has been reached until {retry_after} seconds." - " See http://freshdesk.com/api#ratelimit", - response=response, - ) - elif 500 <= response.status_code < 600: - raise FreshdeskServerError(f"{response.status_code}: Server Error", response=response) - - # Catch any other errors - try: - response.raise_for_status() - except HTTPError as err: - raise FreshdeskError(f"{err}: {body}", response=response) from err - - return body - - @retry_connection_handler(max_tries=5, factor=5) - @retry_after_handler(max_tries=3) - def get(self, url: str, params: Mapping = None): - """Wrapper around request.get() to use the API prefix. Returns a JSON response.""" - params = params or {} - response = self._session.get(self._api_prefix + url, params=params) - return self._parse_and_handle_errors(response) - - def consume_credit(self, credit): - """Consume call credit, if there is no credit left within current window will sleep til next period""" - if self._call_credit: - self._call_credit.consume(credit) - - -class StreamAPI(ABC): - """Basic stream API that allows to iterate over entities""" - - result_return_limit = 100 # maximum value - call_credit = 1 # see https://developers.freshdesk.com/api/#embedding - - def __init__(self, api: API, *args, **kwargs): - super().__init__(*args, **kwargs) - self._api = api - - def _api_get(self, url: str, params: Mapping = None): - """Wrapper around API GET method to respect call rate limit""" - self._api.consume_credit(self.call_credit) - return self._api.get(url, params=params) - - @abstractmethod - def list(self, fields: Sequence[str] = None) -> Iterator[dict]: - """Iterate over entities""" - - def read(self, getter: Callable, params: Mapping[str, Any] = None) -> Iterator: - """Read using getter""" - params = params or {} - - for page in count(start=1): - batch = list( - getter( - params={ - **params, - "per_page": self.result_return_limit, - "page": page, - } - ) - ) - yield from batch - - if len(batch) < self.result_return_limit: - return iter(()) - - -class IncrementalStreamAPI(StreamAPI, ABC): - state_pk = "updated_at" # Name of the field associated with the state - state_filter = "updated_since" # Name of filter that corresponds to the state - - @property - def state(self) -> Optional[Mapping[str, Any]]: - """Current state, if wasn't set return None""" - if self._state: - return {self.state_pk: str(self._state).replace("+00:00", "Z")} - return None - - @state.setter - def state(self, value: Mapping[str, Any]): - self._state = pendulum.parse(value[self.state_pk]) - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._state: Optional[Mapping[str, Any]] = None - - def _state_params(self) -> Mapping[str, Any]: - """Build query parameters responsible for current state""" - if self._state: - return {self.state_filter: self._state} - return {self.state_filter: self._api._start_date} - - @property - def name(self): - """Name of the stream""" - stream_name = self.__class__.__name__ - if stream_name.endswith("API"): - stream_name = stream_name[:-3] - return stream_name - - def read(self, getter: Callable, params: Mapping[str, Any] = None) -> Iterator: - """Read using getter, patched to respect current state""" - params = params or {} - params = {**params, **self._state_params()} - latest_cursor = None - for record in super().read(getter, params): - cursor = pendulum.parse(record[self.state_pk]) - # filter out records older then state - if self._state and self._state >= cursor: - continue - latest_cursor = max(cursor, latest_cursor) if latest_cursor else cursor - yield record - - if latest_cursor: - logger.info(f"Advancing bookmark for {self.name} stream from {self._state} to {latest_cursor}") - self._state = max(latest_cursor, self._state) if self._state else latest_cursor - - -class ClientIncrementalStreamAPI(IncrementalStreamAPI, ABC): - """Incremental stream that don't have native API support, i.e we filter on the client side only""" - - def _state_params(self) -> Mapping[str, Any]: - """Build query parameters responsible for current state, override because API doesn't support this""" - return {} - - -class AgentsAPI(ClientIncrementalStreamAPI): - def list(self, fields: Sequence[str] = None) -> Iterator[dict]: - """Iterate over entities""" - yield from self.read(partial(self._api_get, url="agents")) - - -class CompaniesAPI(ClientIncrementalStreamAPI): - def list(self, fields: Sequence[str] = None) -> Iterator[dict]: - """Iterate over entities""" - yield from self.read(partial(self._api_get, url="companies")) - - -class ContactsAPI(IncrementalStreamAPI): - state_filter = "_updated_since" - - def list(self, fields: Sequence[str] = None) -> Iterator[dict]: - """Iterate over entities""" - yield from self.read(partial(self._api_get, url="contacts")) - - -class GroupsAPI(ClientIncrementalStreamAPI): - """Only users with admin privileges can access the following APIs.""" - - def list(self, fields: Sequence[str] = None) -> Iterator[dict]: - """Iterate over entities""" - yield from self.read(partial(self._api_get, url="groups")) - - -class RolesAPI(ClientIncrementalStreamAPI): - """Only users with admin privileges can access the following APIs.""" - - def list(self, fields: Sequence[str] = None) -> Iterator[dict]: - """Iterate over entities""" - yield from self.read(partial(self._api_get, url="roles")) - - -class SkillsAPI(ClientIncrementalStreamAPI): - """Only users with admin privileges can access the following APIs.""" - - def list(self, fields: Sequence[str] = None) -> Iterator[dict]: - """Iterate over entities""" - yield from self.read(partial(self._api_get, url="skills")) - - -class SurveysAPI(ClientIncrementalStreamAPI): - def list(self, fields: Sequence[str] = None) -> Iterator[dict]: - """Iterate over entities""" - yield from self.read(partial(self._api_get, url="surveys")) - - -class TicketsAPI(IncrementalStreamAPI): - call_credit = 3 # each include consumes 2 additional credits - - def list(self, fields: Sequence[str] = None) -> Iterator[dict]: - """Iterate over entities""" - includes = ["description", "requester", "stats"] - params = {"include": ",".join(includes)} - yield from self.read(partial(self._api_get, url="tickets"), params=params) - - @staticmethod - def get_tickets( - result_return_limit: int, getter: Callable, params: Mapping[str, Any] = None, ticket_paginate_limit: int = 300 - ) -> Iterator: - """ - Read using getter - - This block extends TicketsAPI Stream to overcome '300 page' server error. - Since the TicketsAPI Stream list has a 300 page pagination limit, after 300 pages, update the parameters with - query using 'updated_since' = last_record, if there is more data remaining. - """ - params = params or {} - - # Start page - page = 1 - # Initial request parameters - params = { - **params, - "order_type": "asc", # ASC order, to get the old records first - "order_by": "updated_at", - "per_page": result_return_limit, - } - - while True: - params["page"] = page - batch = list(getter(params=params)) - yield from batch - - if len(batch) < result_return_limit: - return iter(()) - - # checkpoint & switch the pagination - if page == ticket_paginate_limit: - # get last_record from latest batch, pos. -1, because of ACS order of records - last_record_updated_at = batch[-1]["updated_at"] - page = 0 # reset page counter - last_record_updated_at = pendulum.parse(last_record_updated_at) - # updating request parameters with last_record state - params["updated_since"] = last_record_updated_at - # Increment page - page += 1 - - # Override the super().read() method with modified read for tickets - def read(self, getter: Callable, params: Mapping[str, Any] = None) -> Iterator: - """Read using getter, patched to respect current state""" - params = params or {} - params = {**params, **self._state_params()} - latest_cursor = None - for record in self.get_tickets(self.result_return_limit, getter, params): - cursor = pendulum.parse(record[self.state_pk]) - # filter out records older then state - if self._state and self._state >= cursor: - continue - latest_cursor = max(cursor, latest_cursor) if latest_cursor else cursor - yield record - - if latest_cursor: - logger.info(f"Advancing bookmark for {self.name} stream from {self._state} to {latest_cursor}") - self._state = max(latest_cursor, self._state) if self._state else latest_cursor - - -class TimeEntriesAPI(ClientIncrementalStreamAPI): - def list(self, fields: Sequence[str] = None) -> Iterator[dict]: - """Iterate over entities""" - yield from self.read(partial(self._api_get, url="time_entries")) - - -class ConversationsAPI(ClientIncrementalStreamAPI): - """Notes and Replies""" - - def list(self, fields: Sequence[str] = None) -> Iterator[dict]: - """Iterate over entities""" - tickets = TicketsAPI(self._api) - if self.state: - tickets.state = self.state - for ticket in tickets.list(): - url = f"tickets/{ticket['id']}/conversations" - yield from self.read(partial(self._api_get, url=url)) - - -class SatisfactionRatingsAPI(IncrementalStreamAPI): - """Surveys satisfaction replies""" - - state_filter = "created_since" - - def list(self, fields: Sequence[str] = None) -> Iterator[dict]: - """Iterate over entities""" - yield from self.read(partial(self._api_get, url="surveys/satisfaction_ratings")) diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/client.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/client.py deleted file mode 100644 index 9d928337d09f0..0000000000000 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/client.py +++ /dev/null @@ -1,89 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# - - -from airbyte_cdk.entrypoint import logger -from typing import Any, Iterable, Mapping, Tuple - -from airbyte_cdk.models import AirbyteStream -from airbyte_cdk.sources.deprecated.client import BaseClient - -from .api import ( - API, - AgentsAPI, - CompaniesAPI, - ContactsAPI, - ConversationsAPI, - FreshdeskError, - FreshdeskNotFound, - FreshdeskUnauthorized, - GroupsAPI, - RolesAPI, - SatisfactionRatingsAPI, - SkillsAPI, - SurveysAPI, - TicketsAPI, - TimeEntriesAPI, -) - - -class Client(BaseClient): - def __init__(self, domain, api_key, requests_per_minute: int = None, start_date: str = None): - self._api = API(domain=domain, api_key=api_key, requests_per_minute=requests_per_minute, start_date=start_date) - self._apis = { - "agents": AgentsAPI(self._api), - "companies": CompaniesAPI(self._api), - "contacts": ContactsAPI(self._api), - "conversations": ConversationsAPI(self._api), - "groups": GroupsAPI(self._api), - "roles": RolesAPI(self._api), - "skills": SkillsAPI(self._api), - "surveys": SurveysAPI(self._api), - "tickets": TicketsAPI(self._api), - "time_entries": TimeEntriesAPI(self._api), - "satisfaction_ratings": SatisfactionRatingsAPI(self._api), - } - super().__init__() - - @property - def streams(self) -> Iterable[AirbyteStream]: - """List of available streams""" - for stream in super().streams: - if stream.source_defined_cursor: - stream.default_cursor_field = [self._apis[stream.name].state_pk] - yield stream - - def settings(self): - url = "settings/helpdesk" - return self._api.get(url) - - def stream_has_state(self, name: str) -> bool: - """Tell if stream supports incremental sync""" - return hasattr(self._apis[name], "state") - - def get_stream_state(self, name: str) -> Any: - """Get state of stream with corresponding name""" - return self._apis[name].state - - def set_stream_state(self, name: str, state: Any): - """Set state of stream with corresponding name""" - self._apis[name].state = state - - def _enumerate_methods(self) -> Mapping[str, callable]: - return {name: api.list for name, api in self._apis.items()} - - def health_check(self) -> Tuple[bool, str]: - alive = True - error_msg = None - - try: - self.settings() - except (FreshdeskUnauthorized, FreshdeskNotFound): - alive = False - error_msg = "Invalid credentials" - except FreshdeskError as error: - alive = False - error_msg = repr(error) - - return alive, error_msg diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/errors.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/errors.py deleted file mode 100644 index 4173b68648f5f..0000000000000 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/errors.py +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# - - -from requests import HTTPError - - -class FreshdeskError(HTTPError): - """ - Base error class. - Subclassing HTTPError to avoid breaking existing code that expects only HTTPErrors. - """ - - -class FreshdeskBadRequest(FreshdeskError): - """Most 40X and 501 status codes""" - - -class FreshdeskUnauthorized(FreshdeskError): - """401 Unauthorized""" - - -class FreshdeskAccessDenied(FreshdeskError): - """403 Forbidden""" - - -class FreshdeskNotFound(FreshdeskError): - """404""" - - -class FreshdeskRateLimited(FreshdeskError): - """429 Rate Limit Reached""" - - -class FreshdeskServerError(FreshdeskError): - """50X errors""" diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py index 41af7904f4585..8895125a24b65 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py @@ -239,7 +239,7 @@ def read_records( class Conversations(FreshdeskStream): """Notes and Replies""" def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], *args, **kwargs): - super().__init__(**kwargs) + super().__init__(authenticator=authenticator, config=config, args=args, kwargs=kwargs) self.tickets_stream = Tickets(authenticator=authenticator, config=config) def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/utils.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/utils.py index f88669b8a28fe..e6178bfa9351a 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/utils.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/utils.py @@ -2,59 +2,9 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # - -import sys import time -import backoff -import requests from airbyte_cdk.entrypoint import logger -from source_freshdesk.errors import FreshdeskRateLimited - - -def retry_connection_handler(**kwargs): - """Retry helper, log each attempt""" - - def log_retry_attempt(details): - _, exc, _ = sys.exc_info() - logger.info(str(exc)) - logger.info(f"Caught retryable error after {details['tries']} tries. Waiting {details['wait']} more seconds then retrying...") - - def giveup_handler(exc): - return exc.response is not None and 400 <= exc.response.status_code < 500 - - return backoff.on_exception( - backoff.expo, - requests.exceptions.RequestException, - jitter=None, - on_backoff=log_retry_attempt, - giveup=giveup_handler, - **kwargs, - ) - - -def retry_after_handler(**kwargs): - """Retry helper when we hit the call limit, sleeps for specific duration""" - - def sleep_on_ratelimit(_details): - _, exc, _ = sys.exc_info() - if isinstance(exc, FreshdeskRateLimited): - retry_after = int(exc.response.headers["Retry-After"]) - logger.info(f"Rate limit reached. Sleeping for {retry_after} seconds") - time.sleep(retry_after + 1) # extra second to cover any fractions of second - - def log_giveup(_details): - logger.error("Max retry limit reached") - - return backoff.on_exception( - backoff.constant, - FreshdeskRateLimited, - jitter=None, - on_backoff=sleep_on_ratelimit, - on_giveup=log_giveup, - interval=0, # skip waiting part, we will wait in on_backoff handler - **kwargs, - ) class CallCredit: From c8376150e1c0ecfe76d9e3ac75aa8ca9840baf58 Mon Sep 17 00:00:00 2001 From: lgomezm Date: Sun, 24 Apr 2022 23:49:06 -0500 Subject: [PATCH 06/18] Updated test_300_page.py file --- .../integration_tests/test_client.py | 64 -------- .../source_freshdesk/streams.py | 30 ++-- .../unit_tests/test_300_page.py | 137 ++++++++++++------ .../unit_tests/test_client.py | 45 ------ 4 files changed, 116 insertions(+), 160 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-freshdesk/integration_tests/test_client.py delete mode 100644 airbyte-integrations/connectors/source-freshdesk/unit_tests/test_client.py diff --git a/airbyte-integrations/connectors/source-freshdesk/integration_tests/test_client.py b/airbyte-integrations/connectors/source-freshdesk/integration_tests/test_client.py deleted file mode 100644 index a10643db0c462..0000000000000 --- a/airbyte-integrations/connectors/source-freshdesk/integration_tests/test_client.py +++ /dev/null @@ -1,64 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# - - -import json -from pathlib import Path -from typing import Mapping - -import pytest -from source_freshdesk.client import Client - -HERE = Path(__file__).parent.absolute() - - -@pytest.fixture(scope="session") -def account_creds() -> Mapping[str, str]: - config_filename = HERE.parent / "secrets" / "config.json" - - if not config_filename.exists(): - raise RuntimeError(f"Please provide credentials in {config_filename}") - - with open(str(config_filename)) as json_file: - return json.load(json_file) - - -@pytest.fixture -def unknown_account() -> str: - return "unknownaccount.freshdesk.com" - - -@pytest.fixture -def non_freshdesk_account() -> str: - return "unknownaccount.somedomain.com" - - -def test_client_wrong_domain(non_freshdesk_account): - expected_error = "Freshdesk v2 API works only via Freshdesk domains and not via custom CNAMEs" - with pytest.raises(AttributeError, match=expected_error): - Client(domain=non_freshdesk_account, api_key="wrong_key") - - -def test_client_wrong_account(unknown_account): - client = Client(domain=unknown_account, api_key="wrong_key") - alive, error = client.health_check() - - assert not alive - assert error == "Invalid credentials" - - -def test_client_wrong_cred(account_creds): - client = Client(domain=account_creds["domain"], api_key="wrong_key") - alive, error = client.health_check() - - assert not alive - assert error == "Invalid credentials" - - -def test_client_ok(account_creds): - client = Client(domain=account_creds["domain"], api_key=account_creds["api_key"]) - alive, error = client.health_check() - - assert alive - assert not error diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py index 8895125a24b65..95d76bb8b993d 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py @@ -24,6 +24,7 @@ class FreshdeskStream(HttpStream, ABC): """Basic stream API that allows to iterate over entities""" call_credit = 1 # see https://developers.freshdesk.com/api/#embedding + result_return_limit = 100 primary_key = "id" def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], *args, **kwargs): @@ -62,14 +63,17 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, match = LINK_REGEX.search(link_header) next_url = match.group(1) params = parse.parse_qs(parse.urlparse(next_url).query) - return {"per_page": params['per_page'][0], "page": params['page'][0]} + return self.parse_link_params(link_query_params=params) except Exception as e: raise KeyError(f"error parsing next_page token: {e}") + def parse_link_params(self, link_query_params: Mapping[str, List[str]]) -> Mapping[str, Any]: + return {"per_page": link_query_params['per_page'][0], "page": link_query_params['page'][0]} + def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - params = {"per_page": 100} + params = {"per_page": self.result_return_limit} if next_page_token and "page" in next_page_token: params["page"] = next_page_token["page"] return params @@ -99,6 +103,8 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp class IncrementalFreshdeskStream(FreshdeskStream, ABC): + state_filter = "updated_since" # Name of filter that corresponds to the state + @property def cursor_field(self) -> str: return "updated_at" @@ -111,7 +117,7 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late current_stream_state_date = pendulum.parse(current_stream_state_date) latest_record_date = pendulum.parse(latest_record.get("updated_at")) if latest_record.get("updated_at") else self.start_date - return {"updated_at": max(current_stream_state_date, latest_record_date)} + return {"updated_at": max(current_stream_state_date, latest_record_date).strftime("%Y-%m-%dT%H:%M:%SZ")} def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None @@ -168,7 +174,6 @@ def path(self, **kwargs) -> str: class Tickets(IncrementalFreshdeskStream): - state_filter = "updated_since" ticket_paginate_limit = 300 def path(self, **kwargs) -> str: @@ -178,10 +183,20 @@ def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + params.update({ + "order_type": "asc", # ASC order, to get the old records first + "order_by": "updated_at", + }) if next_page_token and "updated_since" in next_page_token: params["updated_since"] = next_page_token["updated_since"] return params + def parse_link_params(self, link_query_params: Mapping[str, List[str]]) -> Mapping[str, Any]: + params = super().parse_link_params(link_query_params) + if "updated_since" in link_query_params: + params["updated_since"] = link_query_params["updated_since"][0] + return params + def read_records( self, sync_mode: SyncMode, @@ -200,7 +215,6 @@ def read_records( pagination_complete = False next_page_token = None - page = 1 with AirbyteSentry.start_transaction("read_records", self.name), AirbyteSentry.start_transaction_span("read_records"): while not pagination_complete: request_headers = self.request_headers( @@ -219,15 +233,13 @@ def read_records( next_page_token = self.next_page_token(response) # checkpoint & switch the pagination - if page == self.ticket_paginate_limit and next_page_token: + if next_page_token and int(next_page_token["page"]) > self.ticket_paginate_limit: # get last_record from latest batch, pos. -1, because of ACS order of records last_record_updated_at = tickets[-1]["updated_at"] - page = 0 # reset page counter last_record_updated_at = pendulum.parse(last_record_updated_at) # updating request parameters with last_record state next_page_token["updated_since"] = last_record_updated_at - # Increment page - page += 1 + del next_page_token["page"] if not next_page_token: pagination_complete = True diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_300_page.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_300_page.py index a8c3a9cc255cc..0387f9512976f 100644 --- a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_300_page.py +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_300_page.py @@ -2,50 +2,81 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # -import pendulum -from source_freshdesk.api import TicketsAPI +import pytest +from requests.auth import HTTPBasicAuth +from airbyte_cdk.models import SyncMode +from source_freshdesk.streams import Tickets -class Test300PageLimit: - - tickets_input = [ - {"id": 1, "updated_at": "2018-01-02T00:00:00Z"}, - {"id": 2, "updated_at": "2018-02-02T00:00:00Z"}, - {"id": 3, "updated_at": "2018-03-02T00:00:00Z"}, - {"id": 4, "updated_at": "2019-01-03T00:00:00Z"}, - {"id": 5, "updated_at": "2019-02-03T00:00:00Z"}, - {"id": 6, "updated_at": "2019-03-03T00:00:00Z"}, - ] - expected_output = [ - {"id": 1, "updated_at": "2018-01-02T00:00:00Z"}, - {"id": 2, "updated_at": "2018-02-02T00:00:00Z"}, - {"id": 2, "updated_at": "2018-02-02T00:00:00Z"}, # duplicate - {"id": 3, "updated_at": "2018-03-02T00:00:00Z"}, - {"id": 3, "updated_at": "2018-03-02T00:00:00Z"}, # duplicate - {"id": 4, "updated_at": "2019-01-03T00:00:00Z"}, - {"id": 4, "updated_at": "2019-01-03T00:00:00Z"}, # duplicate - {"id": 5, "updated_at": "2019-02-03T00:00:00Z"}, - {"id": 5, "updated_at": "2019-02-03T00:00:00Z"}, # duplicate - {"id": 6, "updated_at": "2019-03-03T00:00:00Z"}, - {"id": 6, "updated_at": "2019-03-03T00:00:00Z"}, # duplicate - ] - - # Mocking the getter: Callable to produce the server output - def _getter(self, params, **args): +@pytest.fixture(name="config") +def config_fixture(): + return {"domain": "test.freshdesk.com", "api_key": "secret_api_key", "requests_per_minute": 50, "start_date": "2002-02-10T22:21:44Z"} - tickets_stream = self.tickets_input - updated_since = params.get("updated_since", None) - if updated_since: - tickets_stream = filter(lambda ticket: pendulum.parse(ticket["updated_at"]) >= updated_since, self.tickets_input) +@pytest.fixture(name="authenticator") +def authenticator_fixture(config): + return HTTPBasicAuth(username=config["api_key"], password="unused_with_api_key") - start_from = (params["page"] - 1) * params["per_page"] - output = list(tickets_stream)[start_from : start_from + params["per_page"]] - return output +@pytest.fixture(name="responses") +def responses_fixtures(): + return [ + { + "url": "https://test.freshdesk.com/api/tickets?per_page=1&updated_since=2002-02-10T22%3A21%3A44%2B00%3A00", + "json": [{"id": 1, "updated_at": "2018-01-02T00:00:00Z"}], + "headers": {"Link": '; rel="next"'} + }, + { + "url": "https://test.freshdesk.com/api/tickets?per_page=1&page=2&updated_since=2002-02-10T22%3A21%3A44%2B00%3A00", + "json": [{"id": 2, "updated_at": "2018-02-02T00:00:00Z"}], + "headers": {"Link": '; rel="next"'} + }, + { + "url": "https://test.freshdesk.com/api/tickets?per_page=1&updated_since=2018-02-02T00%3A00%3A00%2B00%3A00", + "json": [{"id": 2, "updated_at": "2018-02-02T00:00:00Z"}], + "headers": {"Link": '; rel="next"'}, + }, + { + "url": "https://test.freshdesk.com/api/tickets?per_page=1&page=2&updated_since=2018-02-02T00%3A00%3A00%2B00%3A00", + "json": [{"id": 3, "updated_at": "2018-03-02T00:00:00Z"}], + "headers": {"Link": '; rel="next"'}, + }, + { + "url": "https://test.freshdesk.com/api/tickets?per_page=1&updated_since=2018-03-02T00%3A00%3A00%2B00%3A00", + "json": [{"id": 3, "updated_at": "2018-03-02T00:00:00Z"}], + "headers": {"Link": '; rel="next"'}, + }, + { + "url": "https://test.freshdesk.com/api/tickets?per_page=1&page=2&updated_since=2018-03-02T00%3A00%3A00%2B00%3A00", + "json": [{"id": 4, "updated_at": "2019-01-03T00:00:00Z"}], + "headers": {"Link": '; rel="next"'}, + }, + { + "url": "https://test.freshdesk.com/api/tickets?per_page=1&updated_since=2019-01-03T00%3A00%3A00%2B00%3A00", + "json": [{"id": 4, "updated_at": "2019-01-03T00:00:00Z"}], + "headers": {"Link": '; rel="next"'}, + }, + { + "url": "https://test.freshdesk.com/api/tickets?per_page=1&page=2&updated_since=2019-01-03T00%3A00%3A00%2B00%3A00", + "json": [{"id": 5, "updated_at": "2019-02-03T00:00:00Z"}], + "headers": {"Link": '; rel="next"'}, + }, + { + "url": "https://test.freshdesk.com/api/tickets?per_page=1&updated_since=2019-02-03T00%3A00%3A00%2B00%3A00", + "json": [{"id": 5, "updated_at": "2019-02-03T00:00:00Z"}], + "headers": {"Link": '; rel="next"'}, + }, + { + "url": "https://test.freshdesk.com/api/tickets?per_page=1&page=2&updated_since=2019-02-03T00%3A00%3A00%2B00%3A00", + "json": [{"id": 6, "updated_at": "2019-03-03T00:00:00Z"}] + } + ] + - def test_not_all_records(self): +class Test300PageLimit: + + def test_not_all_records(self, requests_mock, authenticator, config, responses): """ TEST 1 - not all records are retrieved @@ -64,16 +95,38 @@ def test_not_all_records(self): Main pricipal here is: airbyte is at-least-once delivery, but skipping records is data loss. """ + expected_output = [ + {"id": 1, "updated_at": "2018-01-02T00:00:00Z"}, + {"id": 2, "updated_at": "2018-02-02T00:00:00Z"}, + {"id": 2, "updated_at": "2018-02-02T00:00:00Z"}, # duplicate + {"id": 3, "updated_at": "2018-03-02T00:00:00Z"}, + {"id": 3, "updated_at": "2018-03-02T00:00:00Z"}, # duplicate + {"id": 4, "updated_at": "2019-01-03T00:00:00Z"}, + {"id": 4, "updated_at": "2019-01-03T00:00:00Z"}, # duplicate + {"id": 5, "updated_at": "2019-02-03T00:00:00Z"}, + {"id": 5, "updated_at": "2019-02-03T00:00:00Z"}, # duplicate + {"id": 6, "updated_at": "2019-03-03T00:00:00Z"}, + ] + # INT value of page number where the switch state should be triggered. # in this test case values from: 1 - 4, assuming we want to switch state on this page. ticket_paginate_limit = 2 # This parameter mocks the "per_page" parameter in the API Call result_return_limit = 1 - # Calling the TicketsAPI.get_tickets method directly from the module - test1 = list( - TicketsAPI.get_tickets( - result_return_limit=result_return_limit, getter=self._getter, ticket_paginate_limit=ticket_paginate_limit + + # Create test_stream instance. + test_stream = Tickets(authenticator=authenticator, config=config) + test_stream.ticket_paginate_limit = ticket_paginate_limit + test_stream.result_return_limit = result_return_limit + + # Mocking Request + for response in responses: + requests_mock.register_uri("GET", response["url"], + json=response["json"], + headers=response.get("headers", {}), ) - ) + + records = list(test_stream.read_records(sync_mode=SyncMode.full_refresh)) + # We're expecting 6 records to return from the tickets_stream - assert self.expected_output == test1 + assert records == expected_output diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_client.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_client.py deleted file mode 100644 index 9219fd64be9a4..0000000000000 --- a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_client.py +++ /dev/null @@ -1,45 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# - - -from pathlib import Path - -from pytest import fixture -from source_freshdesk.client import Client - -HERE = Path(__file__).parent.absolute() - - -@fixture(autouse=True) -def time_sleep_mock(mocker): - time_mock = mocker.patch("time.sleep", lambda x: None) - yield time_mock - - -def test_client_backoff_on_limit_reached(requests_mock): - """Error once, check that we retry and not fail""" - responses = [ - {"json": {"error": "limit reached"}, "status_code": 429, "headers": {"Retry-After": "0"}}, - {"json": {"status": "ok"}, "status_code": 200}, - ] - requests_mock.register_uri("GET", "/api/v2/settings/helpdesk", responses) - client = Client(domain="someaccount.freshdesk.com", api_key="somekey") - - result = client.settings() - - assert result == {"status": "ok"} - - -def test_client_backoff_on_server_error(requests_mock): - """Error once, check that we retry and not fail""" - responses = [ - {"json": {"error": "something bad"}, "status_code": 500}, - {"json": {"status": "ok"}, "status_code": 200}, - ] - requests_mock.register_uri("GET", "/api/v2/settings/helpdesk", responses) - client = Client(domain="someaccount.freshdesk.com", api_key="somekey") - - result = client.settings() - - assert result == {"status": "ok"} From a9f760efbace0a96fe169904848805b8c15495c3 Mon Sep 17 00:00:00 2001 From: lgomezm Date: Mon, 25 Apr 2022 21:31:11 -0500 Subject: [PATCH 07/18] Added source tests --- .../source-freshdesk/unit_tests/conftest.py | 16 +++++ .../unit_tests/test_300_page.py | 31 +++------ .../unit_tests/test_source.py | 69 +++++++++++++++++++ 3 files changed, 95 insertions(+), 21 deletions(-) create mode 100644 airbyte-integrations/connectors/source-freshdesk/unit_tests/conftest.py create mode 100644 airbyte-integrations/connectors/source-freshdesk/unit_tests/test_source.py diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/conftest.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/conftest.py new file mode 100644 index 0000000000000..e138fe0d1ff1a --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/conftest.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import pytest +from requests.auth import HTTPBasicAuth + + +@pytest.fixture(name="config") +def config_fixture(): + return {"domain": "test.freshdesk.com", "api_key": "secret_api_key", "requests_per_minute": 50, "start_date": "2002-02-10T22:21:44Z"} + + +@pytest.fixture(name="authenticator") +def authenticator_fixture(config): + return HTTPBasicAuth(username=config["api_key"], password="unused_with_api_key") diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_300_page.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_300_page.py index 0387f9512976f..f92b8fe644c03 100644 --- a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_300_page.py +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_300_page.py @@ -3,72 +3,61 @@ # import pytest -from requests.auth import HTTPBasicAuth from airbyte_cdk.models import SyncMode from source_freshdesk.streams import Tickets -@pytest.fixture(name="config") -def config_fixture(): - return {"domain": "test.freshdesk.com", "api_key": "secret_api_key", "requests_per_minute": 50, "start_date": "2002-02-10T22:21:44Z"} - - -@pytest.fixture(name="authenticator") -def authenticator_fixture(config): - return HTTPBasicAuth(username=config["api_key"], password="unused_with_api_key") - - @pytest.fixture(name="responses") def responses_fixtures(): return [ { - "url": "https://test.freshdesk.com/api/tickets?per_page=1&updated_since=2002-02-10T22%3A21%3A44%2B00%3A00", + "url": "/api/tickets?per_page=1&updated_since=2002-02-10T22%3A21%3A44%2B00%3A00", "json": [{"id": 1, "updated_at": "2018-01-02T00:00:00Z"}], "headers": {"Link": '; rel="next"'} }, { - "url": "https://test.freshdesk.com/api/tickets?per_page=1&page=2&updated_since=2002-02-10T22%3A21%3A44%2B00%3A00", + "url": "/api/tickets?per_page=1&page=2&updated_since=2002-02-10T22%3A21%3A44%2B00%3A00", "json": [{"id": 2, "updated_at": "2018-02-02T00:00:00Z"}], "headers": {"Link": '; rel="next"'} }, { - "url": "https://test.freshdesk.com/api/tickets?per_page=1&updated_since=2018-02-02T00%3A00%3A00%2B00%3A00", + "url": "/api/tickets?per_page=1&updated_since=2018-02-02T00%3A00%3A00%2B00%3A00", "json": [{"id": 2, "updated_at": "2018-02-02T00:00:00Z"}], "headers": {"Link": '; rel="next"'}, }, { - "url": "https://test.freshdesk.com/api/tickets?per_page=1&page=2&updated_since=2018-02-02T00%3A00%3A00%2B00%3A00", + "url": "/api/tickets?per_page=1&page=2&updated_since=2018-02-02T00%3A00%3A00%2B00%3A00", "json": [{"id": 3, "updated_at": "2018-03-02T00:00:00Z"}], "headers": {"Link": '; rel="next"'}, }, { - "url": "https://test.freshdesk.com/api/tickets?per_page=1&updated_since=2018-03-02T00%3A00%3A00%2B00%3A00", + "url": "/api/tickets?per_page=1&updated_since=2018-03-02T00%3A00%3A00%2B00%3A00", "json": [{"id": 3, "updated_at": "2018-03-02T00:00:00Z"}], "headers": {"Link": '; rel="next"'}, }, { - "url": "https://test.freshdesk.com/api/tickets?per_page=1&page=2&updated_since=2018-03-02T00%3A00%3A00%2B00%3A00", + "url": "/api/tickets?per_page=1&page=2&updated_since=2018-03-02T00%3A00%3A00%2B00%3A00", "json": [{"id": 4, "updated_at": "2019-01-03T00:00:00Z"}], "headers": {"Link": '; rel="next"'}, }, { - "url": "https://test.freshdesk.com/api/tickets?per_page=1&updated_since=2019-01-03T00%3A00%3A00%2B00%3A00", + "url": "/api/tickets?per_page=1&updated_since=2019-01-03T00%3A00%3A00%2B00%3A00", "json": [{"id": 4, "updated_at": "2019-01-03T00:00:00Z"}], "headers": {"Link": '; rel="next"'}, }, { - "url": "https://test.freshdesk.com/api/tickets?per_page=1&page=2&updated_since=2019-01-03T00%3A00%3A00%2B00%3A00", + "url": "/api/tickets?per_page=1&page=2&updated_since=2019-01-03T00%3A00%3A00%2B00%3A00", "json": [{"id": 5, "updated_at": "2019-02-03T00:00:00Z"}], "headers": {"Link": '; rel="next"'}, }, { - "url": "https://test.freshdesk.com/api/tickets?per_page=1&updated_since=2019-02-03T00%3A00%3A00%2B00%3A00", + "url": "/api/tickets?per_page=1&updated_since=2019-02-03T00%3A00%3A00%2B00%3A00", "json": [{"id": 5, "updated_at": "2019-02-03T00:00:00Z"}], "headers": {"Link": '; rel="next"'}, }, { - "url": "https://test.freshdesk.com/api/tickets?per_page=1&page=2&updated_since=2019-02-03T00%3A00%3A00%2B00%3A00", + "url": "/api/tickets?per_page=1&page=2&updated_since=2019-02-03T00%3A00%3A00%2B00%3A00", "json": [{"id": 6, "updated_at": "2019-03-03T00:00:00Z"}] } ] diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_source.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_source.py new file mode 100644 index 0000000000000..76197cda6fdc0 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_source.py @@ -0,0 +1,69 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import logging +from source_freshdesk import SourceFreshdesk + +logger = logging.getLogger("test_source") + + +def test_check_connection_ok(requests_mock, config): + json_resp = { + "primary_language": "en", + "supported_languages": [], + "portal_languages": [] + } + + requests_mock.register_uri("GET", "/api/v2/settings/helpdesk", json=json_resp) + ok, error_msg = SourceFreshdesk().check_connection(logger, config=config) + + assert ok + assert not error_msg + + +def test_check_connection_invalid_api_key(requests_mock, config): + responses = [{ + "json": { + "code": "invalid_credentials", + "message": "You have to be logged in to perform this action." + }, + "status_code": 401 + }] + + requests_mock.register_uri("GET", "/api/v2/settings/helpdesk", responses) + ok, error_msg = SourceFreshdesk().check_connection(logger, config=config) + + assert not ok + assert error_msg == "invalid_credentials: You have to be logged in to perform this action." + + +def test_check_connection_empty_config(config): + config = {} + + ok, error_msg = SourceFreshdesk().check_connection(logger, config=config) + + assert not ok + assert error_msg + + +def test_check_connection_invalid_config(config): + config.pop("api_key") + + ok, error_msg = SourceFreshdesk().check_connection(logger, config=config) + + assert not ok + assert error_msg + + +def test_check_connection_exception(config): + ok, error_msg = SourceFreshdesk().check_connection(logger, config=config) + + assert not ok + assert error_msg + + +def test_streams(config): + streams = SourceFreshdesk().streams(config) + + assert len(streams) == 10 From 1f0cdbf8852ddec9c9eb0314fdcd6e19bf28db68 Mon Sep 17 00:00:00 2001 From: lgomezm Date: Mon, 25 Apr 2022 22:26:42 -0500 Subject: [PATCH 08/18] Added stream tests --- .../unit_tests/test_streams.py | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py new file mode 100644 index 0000000000000..588cbe228654e --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py @@ -0,0 +1,89 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import random +import pytest +from typing import Any, MutableMapping + +from source_freshdesk.streams import Agents, Companies, Contacts, Conversations, Groups, Roles, SatisfactionRatings, Skills, Tickets, TimeEntries +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams import Stream + + +def _read_full_refresh(stream_instance: Stream): + records = [] + slices = stream_instance.stream_slices(sync_mode=SyncMode.full_refresh) + for slice in slices: + records.extend(list(stream_instance.read_records(stream_slice=slice, sync_mode=SyncMode.full_refresh))) + return records + + +def _read_incremental(stream_instance: Stream, stream_state: MutableMapping[str, Any]): + res = [] + slices = stream_instance.stream_slices(sync_mode=SyncMode.incremental, stream_state=stream_state) + for slice in slices: + records = stream_instance.read_records(sync_mode=SyncMode.incremental, stream_slice=slice, stream_state=stream_state) + for record in records: + stream_state = stream_instance.get_updated_state(stream_state, record) + res.append(record) + return res, stream_state + + +@pytest.mark.parametrize( + "stream, resource", + [ + (Agents, "agents"), + (Companies, "companies"), + (Contacts, "contacts"), + (Groups, "groups"), + (Roles, "roles"), + (Skills, "skills"), + (TimeEntries, "time_entries"), + (SatisfactionRatings, "surveys/satisfaction_ratings"), + ], +) +def test_full_refresh(stream, resource, authenticator, config, requests_mock): + requests_mock.register_uri("GET", f"/api/{resource}", json=[{"id": x} for x in range(25)]) + + stream = stream(authenticator=authenticator, config=config) + records = _read_full_refresh(stream) + + assert len(records) == 25 + + +def test_full_refresh_conversations(authenticator, config, requests_mock): + requests_mock.register_uri("GET", f"/api/tickets", json=[{"id": x} for x in range(5)]) + for i in range(5): + requests_mock.register_uri("GET", f"/api/tickets/{i}/conversations", json=[{"id": x} for x in range(10)]) + + stream = Conversations(authenticator=authenticator, config=config) + records = _read_full_refresh(stream) + + assert len(records) == 50 + + +@pytest.mark.parametrize( + "stream, resource", + [ + (Contacts, "contacts"), + (Tickets, "tickets"), + (SatisfactionRatings, "surveys/satisfaction_ratings"), + ], +) +def test_incremental(stream, resource, authenticator, config, requests_mock): + highest_updated_at = "2022-04-25T22:00:00Z" + other_updated_at = "2022-04-01T00:00:00Z" + highest_index = random.randint(0, 25) + requests_mock.register_uri( + "GET", + f"/api/{resource}", + json=[{"id": x, "updated_at": highest_updated_at if x == highest_index else other_updated_at} for x in range(25)] + ) + + stream = stream(authenticator=authenticator, config=config) + records, state = _read_incremental(stream, {}) + + assert len(records) == 25 + assert "updated_at" in state + assert state["updated_at"] == highest_updated_at From e31f51b977f1d0daf02b43f1b32f3fc9157751c3 Mon Sep 17 00:00:00 2001 From: lgomezm Date: Sun, 24 Apr 2022 23:49:06 -0500 Subject: [PATCH 09/18] Updated test_300_page.py file --- .../source-freshdesk/unit_tests/test_300_page.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_300_page.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_300_page.py index f92b8fe644c03..5c8f495138dcf 100644 --- a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_300_page.py +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_300_page.py @@ -3,11 +3,22 @@ # import pytest +from requests.auth import HTTPBasicAuth from airbyte_cdk.models import SyncMode from source_freshdesk.streams import Tickets +@pytest.fixture(name="config") +def config_fixture(): + return {"domain": "test.freshdesk.com", "api_key": "secret_api_key", "requests_per_minute": 50, "start_date": "2002-02-10T22:21:44Z"} + + +@pytest.fixture(name="authenticator") +def authenticator_fixture(config): + return HTTPBasicAuth(username=config["api_key"], password="unused_with_api_key") + + @pytest.fixture(name="responses") def responses_fixtures(): return [ From 2d541c808df89feb4636b0e2a21fee4d58091905 Mon Sep 17 00:00:00 2001 From: lgomezm Date: Thu, 28 Apr 2022 22:12:20 -0500 Subject: [PATCH 10/18] Added more streams --- .../schemas/business_hours.json | 110 ++++++++++ .../schemas/canned_response_folders.json | 25 +++ .../schemas/canned_responses.json | 57 +++++ .../schemas/discussion_categories.json | 22 ++ .../schemas/discussion_comments.json | 40 ++++ .../schemas/discussion_forums.json | 40 ++++ .../schemas/discussion_topics.json | 52 +++++ .../schemas/email_configs.json | 37 ++++ .../schemas/email_mailboxes.json | 58 +++++ .../source_freshdesk/schemas/products.json | 22 ++ .../schemas/scenario_automations.json | 39 ++++ .../source_freshdesk/schemas/settings.json | 22 ++ .../schemas/sla_policies.json | 198 ++++++++++++++++++ .../schemas/solution_articles.json | 69 ++++++ .../schemas/solution_categories.json | 28 +++ .../schemas/solution_folders.json | 43 ++++ .../schemas/ticket_fields.json | 95 +++++++++ .../source_freshdesk/source.py | 22 +- .../source_freshdesk/streams.py | 162 +++++++++++++- 19 files changed, 1138 insertions(+), 3 deletions(-) create mode 100644 airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/business_hours.json create mode 100644 airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/canned_response_folders.json create mode 100644 airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/canned_responses.json create mode 100644 airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_categories.json create mode 100644 airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_comments.json create mode 100644 airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_forums.json create mode 100644 airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_topics.json create mode 100644 airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/email_configs.json create mode 100644 airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/email_mailboxes.json create mode 100644 airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/products.json create mode 100644 airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/scenario_automations.json create mode 100644 airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/settings.json create mode 100644 airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/sla_policies.json create mode 100644 airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_articles.json create mode 100644 airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_categories.json create mode 100644 airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_folders.json create mode 100644 airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/ticket_fields.json diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/business_hours.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/business_hours.json new file mode 100644 index 0000000000000..15a3f2c352a77 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/business_hours.json @@ -0,0 +1,110 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "description": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "is_default": { + "type": ["null", "boolean"] + }, + "name": { + "type": ["null", "string"] + }, + "time_zone": { + "type": ["null", "string"] + }, + "business_hours": { + "type": ["null", "object"], + "properties": { + "monday": { + "type": ["null", "object"], + "properties": { + "start_time": { + "type": ["null", "string"] + }, + "end_time": { + "type": ["null", "string"] + } + } + }, + "tuesday": { + "type": ["null", "object"], + "properties": { + "start_time": { + "type": ["null", "string"] + }, + "end_time": { + "type": ["null", "string"] + } + } + }, + "wednesday": { + "type": ["null", "object"], + "properties": { + "start_time": { + "type": ["null", "string"] + }, + "end_time": { + "type": ["null", "string"] + } + } + }, + "thursday": { + "type": ["null", "object"], + "properties": { + "start_time": { + "type": ["null", "string"] + }, + "end_time": { + "type": ["null", "string"] + } + } + }, + "friday": { + "type": ["null", "object"], + "properties": { + "start_time": { + "type": ["null", "string"] + }, + "end_time": { + "type": ["null", "string"] + } + } + }, + "saturday": { + "type": ["null", "object"], + "properties": { + "start_time": { + "type": ["null", "string"] + }, + "end_time": { + "type": ["null", "string"] + } + } + }, + "sunday": { + "type": ["null", "object"], + "properties": { + "start_time": { + "type": ["null", "string"] + }, + "end_time": { + "type": ["null", "string"] + } + } + } + } + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" + } + } + } + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/canned_response_folders.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/canned_response_folders.json new file mode 100644 index 0000000000000..506dffa609d6f --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/canned_response_folders.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "personal": { + "type": ["null", "boolean"] + }, + "responses_count": { + "type": ["null", "integer"] + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" + } + } + } + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/canned_responses.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/canned_responses.json new file mode 100644 index 0000000000000..3e5a13bccbea1 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/canned_responses.json @@ -0,0 +1,57 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "folder_id": { + "type": ["null", "integer"] + }, + "content": { + "type": ["null", "string"] + }, + "content_html": { + "type": ["null", "string"] + }, + "attachments": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "content_type": { + "type": ["null", "string"] + }, + "size": { + "type": ["null", "integer"] + }, + "attachment_url": { + "type": ["null", "string"] + }, + "thumb_url": { + "type": ["null", "string"] + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" + } + } + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" + } + } + } + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_categories.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_categories.json new file mode 100644 index 0000000000000..b9d2116eb1ed8 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_categories.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "description": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" + } + } + } + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_comments.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_comments.json new file mode 100644 index 0000000000000..79a10c2220731 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_comments.json @@ -0,0 +1,40 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "answer": { + "type": ["null", "boolean"] + }, + "body": { + "type": ["null", "string"] + }, + "forum_id": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "published": { + "type": ["null", "boolean"] + }, + "spam": { + "type": ["null", "boolean"] + }, + "topic_id": { + "type": ["null", "integer"] + }, + "trash": { + "type": ["null", "boolean"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" + } + } + } + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_forums.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_forums.json new file mode 100644 index 0000000000000..685d43c84cb87 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_forums.json @@ -0,0 +1,40 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "company_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "description": { + "type": ["null", "string"] + }, + "forum_category_id": { + "type": ["null", "integer"] + }, + "forum_type": { + "type": ["null", "integer"] + }, + "forum_visibility": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "posts_count": { + "type": ["null", "integer"] + }, + "topics_count": { + "type": ["null", "integer"] + } + } + } + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_topics.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_topics.json new file mode 100644 index 0000000000000..d7937ce9603e6 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_topics.json @@ -0,0 +1,52 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "forum_id": { + "type": ["null", "integer"] + }, + "hits": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "locked": { + "type": ["null", "boolean"] + }, + "merged_topic_id": { + "type": ["null", "integer"] + }, + "message": { + "type": ["null", "string"] + }, + "posts_count": { + "type": ["null", "integer"] + }, + "replied_by": { + "type": "string" + }, + "stamp_type": { + "type": ["null", "integer"] + }, + "sticky": { + "type": ["null", "boolean"] + }, + "title": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "user_votes": { + "type": ["null", "integer"] + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" + } + } + } + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/email_configs.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/email_configs.json new file mode 100644 index 0000000000000..2cbb675a52e93 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/email_configs.json @@ -0,0 +1,37 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "active": { + "type": ["null", "boolean"] + }, + "group_id": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "primary_role": { + "type": ["null", "boolean"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "reply_email": { + "type": ["null", "string"] + }, + "to_email": { + "type": ["null", "string"] + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" + } + } + } + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/email_mailboxes.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/email_mailboxes.json new file mode 100644 index 0000000000000..e572a0fc7263d --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/email_mailboxes.json @@ -0,0 +1,58 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "name": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "support_email": { + "type": ["null", "string"] + }, + "group_id": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "default_reply_email": { + "type": ["null", "string"] + }, + "mailbox_type": { + "type": ["null", "string"] + }, + "custom_mailbox": { + "type": ["null", "string"] + }, + "access_type": { + "type": ["null", "string"] + }, + "incoming": { + "type": ["null", "string"] + }, + "mail_server": { + "type": ["null", "string"] + }, + "port": { + "type": ["null", "integer"] + }, + "use_ssl": { + "type": ["null", "boolean"] + }, + "delete_from_server": { + "type": ["null", "boolean"] + }, + "authentication": { + "type": ["null", "string"] + }, + "username": { + "type": ["null", "string"] + }, + "password": { + "type": ["null", "string"] + } + } + } + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/products.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/products.json new file mode 100644 index 0000000000000..b9d2116eb1ed8 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/products.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "description": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" + } + } + } + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/scenario_automations.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/scenario_automations.json new file mode 100644 index 0000000000000..daca2d491c649 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/scenario_automations.json @@ -0,0 +1,39 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "description": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "actions": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + }, + "private": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" + } + } + } + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/settings.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/settings.json new file mode 100644 index 0000000000000..c0d0984ae484d --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/settings.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "primary_language": { + "type": ["null", "string"] + }, + "supported_languages": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "portal_languages": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + } + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/sla_policies.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/sla_policies.json new file mode 100644 index 0000000000000..c3cffad30b9f2 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/sla_policies.json @@ -0,0 +1,198 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "active": { + "type": ["null", "boolean"] + }, + "description": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "is_default": { + "type": ["null", "boolean"] + }, + "name": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "sla_target": { + "type": ["null", "object"], + "properties": { + "priority_4": { + "type": ["null", "object"], + "properties": { + "respond_within": { + "type": ["null", "integer"] + }, + "resolve_within": { + "type": ["null", "integer"] + }, + "business_hours": { + "type": ["null", "boolean"] + }, + "escalation_enabled": { + "type": ["null", "boolean"] + } + } + }, + "priority_3": { + "type": ["null", "object"], + "properties": { + "respond_within": { + "type": ["null", "integer"] + }, + "resolve_within": { + "type": ["null", "integer"] + }, + "business_hours": { + "type": ["null", "boolean"] + }, + "escalation_enabled": { + "type": ["null", "boolean"] + } + } + }, + "priority_2": { + "type": ["null", "object"], + "properties": { + "respond_within": { + "type": ["null", "integer"] + }, + "resolve_within": { + "type": ["null", "integer"] + }, + "business_hours": { + "type": ["null", "boolean"] + }, + "escalation_enabled": { + "type": ["null", "boolean"] + } + } + }, + "priority_1": { + "type": ["null", "object"], + "properties": { + "respond_within": { + "type": ["null", "integer"] + }, + "resolve_within": { + "type": ["null", "integer"] + }, + "business_hours": { + "type": ["null", "boolean"] + }, + "escalation_enabled": { + "type": ["null", "boolean"] + } + } + } + } + }, + "applicable_to": { + "type": ["null", "object"], + "properties": { + "company_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "group_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "sources": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "ticket_types": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "product_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + } + } + }, + "escalation": { + "type": ["null", "object"], + "properties": { + "response": { + "type": ["null", "object"], + "properties": { + "escalation_time": { + "type": ["null", "integer"] + }, + "agent_ids": { + "type": ["null", "integer"] + } + } + }, + "resolution": { + "type": ["null", "object"], + "properties": { + "level1": { + "type": ["null", "object"], + "properties": { + "escalation_time": { + "type": ["null", "integer"] + }, + "agent_ids": { + "type": ["null", "integer"] + } + } + }, + "level2": { + "type": ["null", "object"], + "properties": { + "escalation_time": { + "type": ["null", "integer"] + }, + "agent_ids": { + "type": ["null", "integer"] + } + } + }, + "level3": { + "type": ["null", "object"], + "properties": { + "escalation_time": { + "type": ["null", "integer"] + }, + "agent_ids": { + "type": ["null", "integer"] + } + } + }, + "level4": { + "type": ["null", "object"], + "properties": { + "escalation_time": { + "type": ["null", "integer"] + }, + "agent_ids": { + "type": ["null", "integer"] + } + } + } + } + } + } + } + } + } + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_articles.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_articles.json new file mode 100644 index 0000000000000..20f510dd62c11 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_articles.json @@ -0,0 +1,69 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "agent_id": { + "type": ["null", "integer"] + }, + "category_id": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + }, + "description_text": { + "type": ["null", "string"] + }, + "folder_id": { + "type": ["null", "integer"] + }, + "hits": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "integer"] + }, + "seo_data": { + "type": ["null", "object"], + "properties": { + "meta_title": { + "type": ["null", "string"] + }, + "meta_description": { + "type": ["null", "string"] + }, + "meta_keywords": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + }, + "tags": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "thumbs_down": { + "type": ["null", "integer"] + }, + "thumbs_up": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" + } + } + } + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_categories.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_categories.json new file mode 100644 index 0000000000000..1e764a4f2a408 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_categories.json @@ -0,0 +1,28 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "description": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "visible_in_portals": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" + } + } + } + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_folders.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_folders.json new file mode 100644 index 0000000000000..810231e883713 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_folders.json @@ -0,0 +1,43 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "description": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "visibility": { + "type": ["null", "integer"] + }, + "company_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "contact_segment_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "company_segment_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" + } + } + } + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/ticket_fields.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/ticket_fields.json new file mode 100644 index 0000000000000..b33fecf449072 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/ticket_fields.json @@ -0,0 +1,95 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "description": { + "type": ["null", "string"] + }, + "label": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "required_for_closure": { + "type": ["null", "boolean"] + }, + "type": { + "type": ["null", "string"] + }, + "required_for_agents": { + "type": ["null", "boolean"] + }, + "required_for_customers": { + "type": ["null", "boolean"] + }, + "label_for_customers": { + "type": ["null", "string"] + }, + "customers_can_edit": { + "type": ["null", "boolean"] + }, + "displayed_to_customers": { + "type": ["null", "boolean"] + }, + "portal_cc": { + "type": ["null", "boolean"] + }, + "portal_cc_to": { + "type": ["null", "string"] + }, + "choices": { + "type": ["null", "array"] + }, + "is_fsm": { + "type": ["null", "boolean"] + }, + "field_update_in_progress": { + "type": ["null", "boolean"] + }, + "dependent_fields": { + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "label": { + "type": ["null", "string"] + }, + "label_for_customers": { + "type": ["null", "string"] + }, + "level": { + "type": ["null", "integer"] + }, + "ticket_field_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + } + } + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" + } + } + } + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py index 9638546d8d3ad..ae8c549f1c902 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py @@ -9,7 +9,7 @@ from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream -from source_freshdesk.streams import Agents, Companies, Contacts, Conversations, Groups, Roles, SatisfactionRatings, Skills, Tickets, TimeEntries +from source_freshdesk.streams import Agents, BusinessHours, CannedResponseFolders, CannedResponses, Companies, Contacts, Conversations, DiscussionCategories, DiscussionComments, DiscussionForums, DiscussionTopics, EmailConfigs, EmailMailboxes, Groups, Products, Roles, SatisfactionRatings, ScenarioAutomations, Settings, Skills, SlaPolicies, SolutionArticles, SolutionCategories, SolutionFolders, Surveys, TicketFields, Tickets, TimeEntries class SourceFreshdesk(AbstractSource): @@ -40,13 +40,31 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = self._create_authenticator(config["api_key"]) return [ Agents(authenticator=authenticator, config=config), + BusinessHours(authenticator=authenticator, config=config), + CannedResponseFolders(authenticator=authenticator, config=config), + CannedResponses(authenticator=authenticator, config=config), Companies(authenticator=authenticator, config=config), Contacts(authenticator=authenticator, config=config), Conversations(authenticator=authenticator, config=config), + DiscussionCategories(authenticator=authenticator, config=config), + DiscussionComments(authenticator=authenticator, config=config), + DiscussionForums(authenticator=authenticator, config=config), + DiscussionTopics(authenticator=authenticator, config=config), + EmailConfigs(authenticator=authenticator, config=config), + EmailMailboxes(authenticator=authenticator, config=config), Groups(authenticator=authenticator, config=config), + Products(authenticator=authenticator, config=config), Roles(authenticator=authenticator, config=config), + ScenarioAutomations(authenticator=authenticator, config=config), + Settings(authenticator=authenticator, config=config), Skills(authenticator=authenticator, config=config), + SlaPolicies(authenticator=authenticator, config=config), + SolutionArticles(authenticator=authenticator, config=config), + SolutionCategories(authenticator=authenticator, config=config), + SolutionFolders(authenticator=authenticator, config=config), TimeEntries(authenticator=authenticator, config=config), + TicketFields(authenticator=authenticator, config=config), Tickets(authenticator=authenticator, config=config), - SatisfactionRatings(authenticator=authenticator, config=config) + SatisfactionRatings(authenticator=authenticator, config=config), + Surveys(authenticator=authenticator, config=config) ] diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py index 95d76bb8b993d..0d2c637c31e0d 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py @@ -12,7 +12,7 @@ import re from airbyte_cdk.entrypoint import logger # FIXME (Eugene K): use standard logger from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.utils.sentry import AirbyteSentry from requests.auth import AuthBase from source_freshdesk.utils import CallCredit @@ -136,6 +136,31 @@ def path(self, **kwargs) -> str: return "agents" +class BusinessHours(FreshdeskStream): + + def path(self, **kwargs) -> str: + return "business_hours" + + +class CannedResponseFolders(FreshdeskStream): + + def path(self, **kwargs) -> str: + return "canned_response_folders" + + +class CannedResponses(HttpSubStream, FreshdeskStream): + + def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], **kwargs): + super().__init__( + authenticator=authenticator, + config=config, + parent=CannedResponseFolders(authenticator=authenticator, config=config, **kwargs) + ) + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"canned_response_folders/{stream_slice['parent']['id']}/responses" + + class Companies(FreshdeskStream): def path(self, **kwargs) -> str: @@ -149,30 +174,159 @@ def path(self, **kwargs) -> str: return "contacts" +class DiscussionCategories(FreshdeskStream): + + def path(self, **kwargs) -> str: + return "discussions/categories" + + +class DiscussionForums(HttpSubStream, FreshdeskStream): + + def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], **kwargs): + super().__init__( + authenticator=authenticator, + config=config, + parent=DiscussionCategories(authenticator=authenticator, config=config, **kwargs) + ) + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"discussions/categories/{stream_slice['parent']['id']}/forums" + + +class DiscussionTopics(HttpSubStream, FreshdeskStream): + + def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], **kwargs): + super().__init__( + authenticator=authenticator, + config=config, + parent=DiscussionForums(authenticator=authenticator, config=config, **kwargs) + ) + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"discussions/forums/{stream_slice['parent']['id']}/topics" + + +class DiscussionComments(HttpSubStream, FreshdeskStream): + + def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], **kwargs): + super().__init__( + authenticator=authenticator, + config=config, + parent=DiscussionTopics(authenticator=authenticator, config=config, **kwargs) + ) + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"discussions/topics/{stream_slice['parent']['id']}/comments" + + +class EmailConfigs(FreshdeskStream): + + def path(self, **kwargs) -> str: + return "email_configs" + + +class EmailMailboxes(FreshdeskStream): + + def path(self, **kwargs) -> str: + return "email/mailboxes" + + class Groups(FreshdeskStream): def path(self, **kwargs) -> str: return "groups" +class Products(FreshdeskStream): + + def path(self, **kwargs) -> str: + return "products" + + class Roles(FreshdeskStream): def path(self, **kwargs) -> str: return "roles" +class ScenarioAutomations(FreshdeskStream): + + def path(self, **kwargs) -> str: + return "scenario_automations" + + +class Settings(FreshdeskStream): + + def path(self, **kwargs) -> str: + return "settings/helpdesk" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + yield response.json() + + class Skills(FreshdeskStream): def path(self, **kwargs) -> str: return "skills" +class SlaPolicies(FreshdeskStream): + + def path(self, **kwargs) -> str: + return "sla_policies" + + +class SolutionCategories(FreshdeskStream): + + def path(self, **kwargs) -> str: + return "solutions/categories" + + +class SolutionFolders(HttpSubStream, FreshdeskStream): + + def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], **kwargs): + super().__init__( + authenticator=authenticator, + config=config, + parent=SolutionCategories(authenticator=authenticator, config=config, **kwargs) + ) + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"solutions/categories/{stream_slice['parent']['id']}/folders" + + def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping]: + records = response.json() + category_id = stream_slice['parent']['id'] + for record in records: + record.setdefault("category_id", category_id) + yield record + + +class SolutionArticles(HttpSubStream, FreshdeskStream): + + def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], **kwargs): + super().__init__( + authenticator=authenticator, + config=config, + parent=SolutionFolders(authenticator=authenticator, config=config, **kwargs) + ) + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"solutions/folders/{stream_slice['parent']['id']}/articles" + + class TimeEntries(FreshdeskStream): def path(self, **kwargs) -> str: return "time_entries" +class TicketFields(FreshdeskStream): + + def path(self, **kwargs) -> str: + return "ticket_fields" + + class Tickets(IncrementalFreshdeskStream): ticket_paginate_limit = 300 @@ -271,3 +425,9 @@ class SatisfactionRatings(IncrementalFreshdeskStream): def path(self, **kwargs) -> str: return "surveys/satisfaction_ratings" + + +class Surveys(FreshdeskStream): + + def path(self, **kwargs) -> str: + return "surveys" From 8a95b1f322acb2157bdace199bc29a2eaf5bd77b Mon Sep 17 00:00:00 2001 From: lgomezm Date: Thu, 28 Apr 2022 22:12:35 -0500 Subject: [PATCH 11/18] Added tests for new streams --- .../unit_tests/test_source.py | 2 +- .../unit_tests/test_streams.py | 91 +++++++++++++++++-- 2 files changed, 85 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_source.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_source.py index 76197cda6fdc0..1ad36f77b18ec 100644 --- a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_source.py @@ -66,4 +66,4 @@ def test_check_connection_exception(config): def test_streams(config): streams = SourceFreshdesk().streams(config) - assert len(streams) == 10 + assert len(streams) == 28 diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py index 588cbe228654e..fc6ad2dca9638 100644 --- a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py @@ -6,7 +6,14 @@ import pytest from typing import Any, MutableMapping -from source_freshdesk.streams import Agents, Companies, Contacts, Conversations, Groups, Roles, SatisfactionRatings, Skills, Tickets, TimeEntries +from source_freshdesk.streams import ( + Agents, BusinessHours, CannedResponseFolders, CannedResponses, Companies, + Contacts, Conversations, DiscussionCategories, DiscussionComments, + DiscussionForums, DiscussionTopics, EmailConfigs, EmailMailboxes, Groups, + Products, Roles, SatisfactionRatings, ScenarioAutomations, Settings, Skills, + SlaPolicies, SolutionArticles, SolutionCategories, SolutionFolders, Surveys, + TicketFields, Tickets, TimeEntries +) from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import Stream @@ -41,6 +48,17 @@ def _read_incremental(stream_instance: Stream, stream_state: MutableMapping[str, (Skills, "skills"), (TimeEntries, "time_entries"), (SatisfactionRatings, "surveys/satisfaction_ratings"), + (BusinessHours, "business_hours"), + (CannedResponseFolders, "canned_response_folders"), + (DiscussionCategories, "discussions/categories"), + (EmailConfigs, "email_configs"), + (EmailMailboxes, "email/mailboxes"), + (Products, "products"), + (ScenarioAutomations, "scenario_automations"), + (SlaPolicies, "sla_policies"), + (SolutionCategories, "solutions/categories"), + (TicketFields, "ticket_fields"), + (Surveys, "surveys"), ], ) def test_full_refresh(stream, resource, authenticator, config, requests_mock): @@ -52,15 +70,19 @@ def test_full_refresh(stream, resource, authenticator, config, requests_mock): assert len(records) == 25 -def test_full_refresh_conversations(authenticator, config, requests_mock): - requests_mock.register_uri("GET", f"/api/tickets", json=[{"id": x} for x in range(5)]) - for i in range(5): - requests_mock.register_uri("GET", f"/api/tickets/{i}/conversations", json=[{"id": x} for x in range(10)]) +def test_full_refresh_settings(authenticator, config, requests_mock): + json_resp = { + "primary_language": "en", + "supported_languages": [], + "portal_languages": [] + } + requests_mock.register_uri("GET", f"/api/settings/helpdesk", json=json_resp) - stream = Conversations(authenticator=authenticator, config=config) + stream = Settings(authenticator=authenticator, config=config) records = _read_full_refresh(stream) - assert len(records) == 50 + assert len(records) == 1 + assert records[0] == json_resp @pytest.mark.parametrize( @@ -87,3 +109,58 @@ def test_incremental(stream, resource, authenticator, config, requests_mock): assert len(records) == 25 assert "updated_at" in state assert state["updated_at"] == highest_updated_at + + +@pytest.mark.parametrize( + "stream_class, parent_path, sub_paths", + [ + (CannedResponses, 'canned_response_folders', [f"canned_response_folders/{x}/responses" for x in range(5)]), + (Conversations, 'tickets', [f"tickets/{x}/conversations" for x in range(5)]), + (DiscussionForums, 'discussions/categories', [f"discussions/categories/{x}/forums" for x in range(5)]), + (SolutionFolders, 'solutions/categories', [f"solutions/categories/{x}/folders" for x in range(5)]), + ] +) +def test_substream_full_refresh(requests_mock, stream_class, parent_path, sub_paths, authenticator, config): + requests_mock.register_uri("GET", "/api/"+parent_path, json=[{"id": x} for x in range(5)]) + for sub_path in sub_paths: + requests_mock.register_uri("GET", "/api/"+sub_path, json=[{"id": x} for x in range(10)]) + + stream = stream_class(authenticator=authenticator, config=config) + records = _read_full_refresh(stream) + + assert len(records) == 50 + + +@pytest.mark.parametrize( + "stream_class, parent_path, sub_paths, sub_sub_paths", + [ + (DiscussionTopics, 'discussions/categories', [f"discussions/categories/{x}/forums" for x in range(5)], [f"discussions/forums/{x}/topics" for x in range(5)]), + (SolutionArticles, 'solutions/categories', [f"solutions/categories/{x}/folders" for x in range(5)], [f"solutions/folders/{x}/articles" for x in range(5)]), + ] +) +def test_full_refresh_with_two_sub_levels(requests_mock, stream_class, parent_path, sub_paths, sub_sub_paths, authenticator, config): + requests_mock.register_uri("GET", "/api/"+parent_path, json=[{"id": x} for x in range(5)]) + for sub_path in sub_paths: + requests_mock.register_uri("GET", f"/api/"+sub_path, json=[{"id": x} for x in range(5)]) + for sub_sub_path in sub_sub_paths: + requests_mock.register_uri("GET", f"/api/"+sub_sub_path, json=[{"id": x} for x in range(10)]) + + stream = stream_class(authenticator=authenticator, config=config) + records = _read_full_refresh(stream) + + assert len(records) == 250 + + +def test_full_refresh_discussion_comments(requests_mock, authenticator, config): + requests_mock.register_uri("GET", "/api/discussions/categories", json=[{"id": x} for x in range(2)]) + for i in range(2): + requests_mock.register_uri("GET", f"/api/discussions/categories/{i}/forums", json=[{"id": x} for x in range(3)]) + for j in range(3): + requests_mock.register_uri("GET", f"/api/discussions/forums/{j}/topics", json=[{"id": x} for x in range(4)]) + for k in range(4): + requests_mock.register_uri("GET", f"/api/discussions/topics/{k}/comments", json=[{"id": x} for x in range(5)]) + + stream = DiscussionComments(authenticator=authenticator, config=config) + records = _read_full_refresh(stream) + + assert len(records) == 120 From 3c72a9fba460db93a22fd62a64049a29866f5730 Mon Sep 17 00:00:00 2001 From: lgomezm Date: Fri, 29 Apr 2022 22:44:42 -0500 Subject: [PATCH 12/18] Fixed integration tests --- .../acceptance-test-config.yml | 5 +- .../schemas/canned_responses.json | 53 ++++++++++--------- .../schemas/discussion_topics.json | 2 +- .../schemas/email_mailboxes.json | 2 +- .../schemas/ticket_fields.json | 2 +- .../source_freshdesk/streams.py | 2 + 6 files changed, 37 insertions(+), 29 deletions(-) diff --git a/airbyte-integrations/connectors/source-freshdesk/acceptance-test-config.yml b/airbyte-integrations/connectors/source-freshdesk/acceptance-test-config.yml index a3e6ff797dacd..967bab02af05e 100644 --- a/airbyte-integrations/connectors/source-freshdesk/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-freshdesk/acceptance-test-config.yml @@ -13,9 +13,12 @@ tests: - config_path: "secrets/config.json" basic_read: - config_path: "secrets/config.json" - empty_streams: ["satisfaction_ratings", "tickets", "time_entries", "conversations"] + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: ["satisfaction_ratings", "tickets", "time_entries", "conversations", "scenario_automations"] incremental: - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" full_refresh: - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/canned_responses.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/canned_responses.json index 3e5a13bccbea1..9a3f39762775d 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/canned_responses.json +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/canned_responses.json @@ -18,31 +18,34 @@ "type": ["null", "string"] }, "attachments": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "content_type": { - "type": ["null", "string"] - }, - "size": { - "type": ["null", "integer"] - }, - "attachment_url": { - "type": ["null", "string"] - }, - "thumb_url": { - "type": ["null", "string"] - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "content_type": { + "type": ["null", "string"] + }, + "size": { + "type": ["null", "integer"] + }, + "attachment_url": { + "type": ["null", "string"] + }, + "thumb_url": { + "type": ["null", "string"] + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" + } } } }, diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_topics.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_topics.json index d7937ce9603e6..f5bf176613d36 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_topics.json +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_topics.json @@ -24,7 +24,7 @@ "type": ["null", "integer"] }, "replied_by": { - "type": "string" + "type": ["null", "integer"] }, "stamp_type": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/email_mailboxes.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/email_mailboxes.json index e572a0fc7263d..0f3d99cef6a13 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/email_mailboxes.json +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/email_mailboxes.json @@ -18,7 +18,7 @@ "type": ["null", "integer"] }, "default_reply_email": { - "type": ["null", "string"] + "type": ["null", "boolean"] }, "mailbox_type": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/ticket_fields.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/ticket_fields.json index b33fecf449072..ae302f14b7a55 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/ticket_fields.json +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/ticket_fields.json @@ -48,7 +48,7 @@ "type": ["null", "string"] }, "choices": { - "type": ["null", "array"] + "type": ["null", "array", "object"] }, "is_fsm": { "type": ["null", "boolean"] diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py index 0d2c637c31e0d..57cc8592fff68 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py @@ -257,6 +257,8 @@ def path(self, **kwargs) -> str: class Settings(FreshdeskStream): + primary_key = "primary_language" + def path(self, **kwargs) -> str: return "settings/helpdesk" From 5b1e4627d4e30ec3011f7c399c001ddc64364988 Mon Sep 17 00:00:00 2001 From: lgomezm Date: Fri, 29 Apr 2022 22:45:25 -0500 Subject: [PATCH 13/18] Added integrtation test catalog --- .../integration_tests/configured_catalog.json | 245 ++++++++++++++++++ 1 file changed, 245 insertions(+) create mode 100644 airbyte-integrations/connectors/source-freshdesk/integration_tests/configured_catalog.json diff --git a/airbyte-integrations/connectors/source-freshdesk/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-freshdesk/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..d510e7ff18422 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/integration_tests/configured_catalog.json @@ -0,0 +1,245 @@ +{ + "streams": [ + { + "stream": { + "name": "agents", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "business_hours", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "canned_response_folders", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "canned_responses", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "companies", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "contacts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "discussion_categories", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "discussion_forums", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "discussion_topics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "discussion_comments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "email_configs", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "email_mailboxes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "groups", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "roles", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "scenario_automations", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "settings", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "sla_policies", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "solution_categories", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "solution_folders", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "solution_articles", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "time_entries", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ticket_fields", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "tickets", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "conversations", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "satisfaction_ratings", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "surveys", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + } + ] + } + \ No newline at end of file From 280e986ccd567017948dca352d64e1b7533447b6 Mon Sep 17 00:00:00 2001 From: lgomezm Date: Tue, 31 May 2022 21:46:58 -0500 Subject: [PATCH 14/18] Fixed formatting --- .../source_freshdesk/source.py | 4 +- .../source_freshdesk/streams.py | 35 +++++---------- .../unit_tests/test_source.py | 1 + .../unit_tests/test_streams.py | 44 +++++++++++-------- 4 files changed, 38 insertions(+), 46 deletions(-) diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py index fab301c7f2e99..ed5cec27e1c44 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py @@ -68,7 +68,7 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> error_msg = repr(error) return alive, error_msg - + def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = FreshdeskAuth(config["api_key"]) stream_kwargs = {"authenticator": authenticator, "config": config} @@ -100,5 +100,5 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: TicketFields(**stream_kwargs), Tickets(**stream_kwargs), SatisfactionRatings(**stream_kwargs), - Surveys(**stream_kwargs) + Surveys(**stream_kwargs), ] diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py index ad25836ffa551..3adad7d1e6ced 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py @@ -146,12 +146,9 @@ def path(self, **kwargs) -> str: class CannedResponses(HttpSubStream, FreshdeskStream): - def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], **kwargs): super().__init__( - authenticator=authenticator, - config=config, - parent=CannedResponseFolders(authenticator=authenticator, config=config, **kwargs) + authenticator=authenticator, config=config, parent=CannedResponseFolders(authenticator=authenticator, config=config, **kwargs) ) def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: @@ -176,12 +173,9 @@ def path(self, **kwargs) -> str: class DiscussionForums(HttpSubStream, FreshdeskStream): - def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], **kwargs): super().__init__( - authenticator=authenticator, - config=config, - parent=DiscussionCategories(authenticator=authenticator, config=config, **kwargs) + authenticator=authenticator, config=config, parent=DiscussionCategories(authenticator=authenticator, config=config, **kwargs) ) def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: @@ -189,12 +183,9 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class DiscussionTopics(HttpSubStream, FreshdeskStream): - def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], **kwargs): super().__init__( - authenticator=authenticator, - config=config, - parent=DiscussionForums(authenticator=authenticator, config=config, **kwargs) + authenticator=authenticator, config=config, parent=DiscussionForums(authenticator=authenticator, config=config, **kwargs) ) def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: @@ -202,12 +193,9 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class DiscussionComments(HttpSubStream, FreshdeskStream): - def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], **kwargs): super().__init__( - authenticator=authenticator, - config=config, - parent=DiscussionTopics(authenticator=authenticator, config=config, **kwargs) + authenticator=authenticator, config=config, parent=DiscussionTopics(authenticator=authenticator, config=config, **kwargs) ) def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: @@ -249,7 +237,7 @@ class Settings(FreshdeskStream): def path(self, **kwargs) -> str: return "settings/helpdesk" - + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: yield response.json() @@ -272,17 +260,15 @@ def path(self, **kwargs) -> str: class SolutionFolders(HttpSubStream, FreshdeskStream): def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], **kwargs): super().__init__( - authenticator=authenticator, - config=config, - parent=SolutionCategories(authenticator=authenticator, config=config, **kwargs) + authenticator=authenticator, config=config, parent=SolutionCategories(authenticator=authenticator, config=config, **kwargs) ) def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: return f"solutions/categories/{stream_slice['parent']['id']}/folders" - + def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping]: records = response.json() - category_id = stream_slice['parent']['id'] + category_id = stream_slice["parent"]["id"] for record in records: record.setdefault("category_id", category_id) yield record @@ -291,9 +277,7 @@ def parse_response(self, response: requests.Response, stream_slice: Mapping[str, class SolutionArticles(HttpSubStream, FreshdeskStream): def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], **kwargs): super().__init__( - authenticator=authenticator, - config=config, - parent=SolutionFolders(authenticator=authenticator, config=config, **kwargs) + authenticator=authenticator, config=config, parent=SolutionFolders(authenticator=authenticator, config=config, **kwargs) ) def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: @@ -357,6 +341,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, class Conversations(FreshdeskStream): """Notes and Replies""" + def __init__(self, authenticator: AuthBase, config: Mapping[str, Any], *args, **kwargs): super().__init__(authenticator=authenticator, config=config, args=args, kwargs=kwargs) self.tickets_stream = Tickets(authenticator=authenticator, config=config) diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_source.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_source.py index 4e94ea15fbd11..3aba345f653ec 100644 --- a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_source.py @@ -3,6 +3,7 @@ # import logging + from source_freshdesk import SourceFreshdesk logger = logging.getLogger("test_source") diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py index e1291241eae64..7340eae28e8bb 100644 --- a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py @@ -103,12 +103,8 @@ def test_full_refresh_conversations(authenticator, config, requests_mock): def test_full_refresh_settings(authenticator, config, requests_mock): - json_resp = { - "primary_language": "en", - "supported_languages": [], - "portal_languages": [] - } - requests_mock.register_uri("GET", f"/api/settings/helpdesk", json=json_resp) + json_resp = {"primary_language": "en", "supported_languages": [], "portal_languages": []} + requests_mock.register_uri("GET", "/api/settings/helpdesk", json=json_resp) stream = Settings(authenticator=authenticator, config=config) records = _read_full_refresh(stream) @@ -146,16 +142,16 @@ def test_incremental(stream, resource, authenticator, config, requests_mock): @pytest.mark.parametrize( "stream_class, parent_path, sub_paths", [ - (CannedResponses, 'canned_response_folders', [f"canned_response_folders/{x}/responses" for x in range(5)]), - (Conversations, 'tickets', [f"tickets/{x}/conversations" for x in range(5)]), - (DiscussionForums, 'discussions/categories', [f"discussions/categories/{x}/forums" for x in range(5)]), - (SolutionFolders, 'solutions/categories', [f"solutions/categories/{x}/folders" for x in range(5)]), - ] + (CannedResponses, "canned_response_folders", [f"canned_response_folders/{x}/responses" for x in range(5)]), + (Conversations, "tickets", [f"tickets/{x}/conversations" for x in range(5)]), + (DiscussionForums, "discussions/categories", [f"discussions/categories/{x}/forums" for x in range(5)]), + (SolutionFolders, "solutions/categories", [f"solutions/categories/{x}/folders" for x in range(5)]), + ], ) def test_substream_full_refresh(requests_mock, stream_class, parent_path, sub_paths, authenticator, config): - requests_mock.register_uri("GET", "/api/"+parent_path, json=[{"id": x, "updated_at": "2022-05-05T00:00:00Z"} for x in range(5)]) + requests_mock.register_uri("GET", "/api/" + parent_path, json=[{"id": x, "updated_at": "2022-05-05T00:00:00Z"} for x in range(5)]) for sub_path in sub_paths: - requests_mock.register_uri("GET", "/api/"+sub_path, json=[{"id": x, "updated_at": "2022-05-05T00:00:00Z"} for x in range(10)]) + requests_mock.register_uri("GET", "/api/" + sub_path, json=[{"id": x, "updated_at": "2022-05-05T00:00:00Z"} for x in range(10)]) stream = stream_class(authenticator=authenticator, config=config) records = _read_full_refresh(stream) @@ -166,16 +162,26 @@ def test_substream_full_refresh(requests_mock, stream_class, parent_path, sub_pa @pytest.mark.parametrize( "stream_class, parent_path, sub_paths, sub_sub_paths", [ - (DiscussionTopics, 'discussions/categories', [f"discussions/categories/{x}/forums" for x in range(5)], [f"discussions/forums/{x}/topics" for x in range(5)]), - (SolutionArticles, 'solutions/categories', [f"solutions/categories/{x}/folders" for x in range(5)], [f"solutions/folders/{x}/articles" for x in range(5)]), - ] + ( + DiscussionTopics, + "discussions/categories", + [f"discussions/categories/{x}/forums" for x in range(5)], + [f"discussions/forums/{x}/topics" for x in range(5)], + ), + ( + SolutionArticles, + "solutions/categories", + [f"solutions/categories/{x}/folders" for x in range(5)], + [f"solutions/folders/{x}/articles" for x in range(5)], + ), + ], ) def test_full_refresh_with_two_sub_levels(requests_mock, stream_class, parent_path, sub_paths, sub_sub_paths, authenticator, config): - requests_mock.register_uri("GET", "/api/"+parent_path, json=[{"id": x} for x in range(5)]) + requests_mock.register_uri("GET", f"/api/{parent_path}", json=[{"id": x} for x in range(5)]) for sub_path in sub_paths: - requests_mock.register_uri("GET", f"/api/"+sub_path, json=[{"id": x} for x in range(5)]) + requests_mock.register_uri("GET", f"/api/{sub_path}", json=[{"id": x} for x in range(5)]) for sub_sub_path in sub_sub_paths: - requests_mock.register_uri("GET", f"/api/"+sub_sub_path, json=[{"id": x} for x in range(10)]) + requests_mock.register_uri("GET", f"/api/{sub_sub_path}", json=[{"id": x} for x in range(10)]) stream = stream_class(authenticator=authenticator, config=config) records = _read_full_refresh(stream) From 748d736a1a9312c47069dfe4b3f637450ebb4081 Mon Sep 17 00:00:00 2001 From: lgomezm Date: Wed, 1 Jun 2022 08:58:39 -0500 Subject: [PATCH 15/18] Fixed unit tests --- .../connectors/source-freshdesk/unit_tests/test_streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py index 7340eae28e8bb..de704263bc3df 100644 --- a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py @@ -124,7 +124,7 @@ def test_full_refresh_settings(authenticator, config, requests_mock): def test_incremental(stream, resource, authenticator, config, requests_mock): highest_updated_at = "2022-04-25T22:00:00Z" other_updated_at = "2022-04-01T00:00:00Z" - highest_index = random.randint(0, 25) + highest_index = random.randint(0, 24) requests_mock.register_uri( "GET", f"/api/{resource}", From 02ef0ab2fc59bc289686cb41fc50dfef01f313f9 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 3 Jun 2022 14:29:50 +0200 Subject: [PATCH 16/18] format stream schemas --- .../schemas/business_hours.json | 205 +++++----- .../schemas/canned_response_folders.json | 43 ++- .../schemas/canned_responses.json | 109 +++--- .../schemas/discussion_categories.json | 37 +- .../schemas/discussion_comments.json | 73 ++-- .../schemas/discussion_forums.json | 69 ++-- .../schemas/discussion_topics.json | 97 +++-- .../schemas/email_configs.json | 67 ++-- .../schemas/email_mailboxes.json | 109 +++--- .../source_freshdesk/schemas/products.json | 37 +- .../schemas/scenario_automations.json | 67 ++-- .../source_freshdesk/schemas/settings.json | 33 +- .../schemas/sla_policies.json | 357 +++++++++--------- .../schemas/solution_articles.json | 123 +++--- .../schemas/solution_categories.json | 45 ++- .../schemas/solution_folders.json | 73 ++-- .../schemas/ticket_fields.json | 181 +++++---- 17 files changed, 854 insertions(+), 871 deletions(-) diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/business_hours.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/business_hours.json index 15a3f2c352a77..451eb1c3be158 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/business_hours.json +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/business_hours.json @@ -1,110 +1,109 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "description": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "is_default": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "time_zone": { - "type": ["null", "string"] - }, - "business_hours": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "description": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "is_default": { + "type": ["null", "boolean"] + }, + "name": { + "type": ["null", "string"] + }, + "time_zone": { + "type": ["null", "string"] + }, + "business_hours": { + "type": ["null", "object"], + "properties": { + "monday": { "type": ["null", "object"], "properties": { - "monday": { - "type": ["null", "object"], - "properties": { - "start_time": { - "type": ["null", "string"] - }, - "end_time": { - "type": ["null", "string"] - } - } - }, - "tuesday": { - "type": ["null", "object"], - "properties": { - "start_time": { - "type": ["null", "string"] - }, - "end_time": { - "type": ["null", "string"] - } - } - }, - "wednesday": { - "type": ["null", "object"], - "properties": { - "start_time": { - "type": ["null", "string"] - }, - "end_time": { - "type": ["null", "string"] - } - } - }, - "thursday": { - "type": ["null", "object"], - "properties": { - "start_time": { - "type": ["null", "string"] - }, - "end_time": { - "type": ["null", "string"] - } - } - }, - "friday": { - "type": ["null", "object"], - "properties": { - "start_time": { - "type": ["null", "string"] - }, - "end_time": { - "type": ["null", "string"] - } - } - }, - "saturday": { - "type": ["null", "object"], - "properties": { - "start_time": { - "type": ["null", "string"] - }, - "end_time": { - "type": ["null", "string"] - } - } - }, - "sunday": { - "type": ["null", "object"], - "properties": { - "start_time": { - "type": ["null", "string"] - }, - "end_time": { - "type": ["null", "string"] - } - } - } + "start_time": { + "type": ["null", "string"] + }, + "end_time": { + "type": ["null", "string"] + } } - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" + }, + "tuesday": { + "type": ["null", "object"], + "properties": { + "start_time": { + "type": ["null", "string"] + }, + "end_time": { + "type": ["null", "string"] + } + } + }, + "wednesday": { + "type": ["null", "object"], + "properties": { + "start_time": { + "type": ["null", "string"] + }, + "end_time": { + "type": ["null", "string"] + } + } + }, + "thursday": { + "type": ["null", "object"], + "properties": { + "start_time": { + "type": ["null", "string"] + }, + "end_time": { + "type": ["null", "string"] + } + } + }, + "friday": { + "type": ["null", "object"], + "properties": { + "start_time": { + "type": ["null", "string"] + }, + "end_time": { + "type": ["null", "string"] + } + } + }, + "saturday": { + "type": ["null", "object"], + "properties": { + "start_time": { + "type": ["null", "string"] + }, + "end_time": { + "type": ["null", "string"] + } + } + }, + "sunday": { + "type": ["null", "object"], + "properties": { + "start_time": { + "type": ["null", "string"] + }, + "end_time": { + "type": ["null", "string"] + } + } + } } + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" } } - \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/canned_response_folders.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/canned_response_folders.json index 506dffa609d6f..515af892c44d4 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/canned_response_folders.json +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/canned_response_folders.json @@ -1,25 +1,24 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "personal": { - "type": ["null", "boolean"] - }, - "responses_count": { - "type": ["null", "integer"] - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "personal": { + "type": ["null", "boolean"] + }, + "responses_count": { + "type": ["null", "integer"] + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" } } - \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/canned_responses.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/canned_responses.json index 9a3f39762775d..7b95714a3229d 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/canned_responses.json +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/canned_responses.json @@ -1,60 +1,59 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "integer"] - }, - "title": { - "type": ["null", "string"] - }, - "folder_id": { - "type": ["null", "integer"] - }, - "content": { - "type": ["null", "string"] - }, - "content_html": { - "type": ["null", "string"] - }, - "attachments": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "content_type": { - "type": ["null", "string"] - }, - "size": { - "type": ["null", "integer"] - }, - "attachment_url": { - "type": ["null", "string"] - }, - "thumb_url": { - "type": ["null", "string"] - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "folder_id": { + "type": ["null", "integer"] + }, + "content": { + "type": ["null", "string"] + }, + "content_html": { + "type": ["null", "string"] + }, + "attachments": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "content_type": { + "type": ["null", "string"] + }, + "size": { + "type": ["null", "integer"] + }, + "attachment_url": { + "type": ["null", "string"] + }, + "thumb_url": { + "type": ["null", "string"] + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" } - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" + } } + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" } } - \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_categories.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_categories.json index b9d2116eb1ed8..00d4b144b3019 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_categories.json +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_categories.json @@ -1,22 +1,21 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "description": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "description": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" } } - \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_comments.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_comments.json index 79a10c2220731..14a38830b1eb2 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_comments.json +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_comments.json @@ -1,40 +1,39 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "answer": { - "type": ["null", "boolean"] - }, - "body": { - "type": ["null", "string"] - }, - "forum_id": { - "type": ["null", "integer"] - }, - "id": { - "type": ["null", "integer"] - }, - "published": { - "type": ["null", "boolean"] - }, - "spam": { - "type": ["null", "boolean"] - }, - "topic_id": { - "type": ["null", "integer"] - }, - "trash": { - "type": ["null", "boolean"] - }, - "user_id": { - "type": ["null", "integer"] - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "answer": { + "type": ["null", "boolean"] + }, + "body": { + "type": ["null", "string"] + }, + "forum_id": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "published": { + "type": ["null", "boolean"] + }, + "spam": { + "type": ["null", "boolean"] + }, + "topic_id": { + "type": ["null", "integer"] + }, + "trash": { + "type": ["null", "boolean"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" } } - \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_forums.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_forums.json index 685d43c84cb87..5c6eb9d8e9a38 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_forums.json +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_forums.json @@ -1,40 +1,39 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "company_ids": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } - }, - "description": { - "type": ["null", "string"] - }, - "forum_category_id": { - "type": ["null", "integer"] - }, - "forum_type": { - "type": ["null", "integer"] - }, - "forum_visibility": { - "type": ["null", "integer"] - }, - "id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "position": { - "type": ["null", "integer"] - }, - "posts_count": { - "type": ["null", "integer"] - }, - "topics_count": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "company_ids": { + "type": ["null", "array"], + "items": { "type": ["null", "integer"] } + }, + "description": { + "type": ["null", "string"] + }, + "forum_category_id": { + "type": ["null", "integer"] + }, + "forum_type": { + "type": ["null", "integer"] + }, + "forum_visibility": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "posts_count": { + "type": ["null", "integer"] + }, + "topics_count": { + "type": ["null", "integer"] } } - \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_topics.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_topics.json index f5bf176613d36..eceb6f721c07b 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_topics.json +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/discussion_topics.json @@ -1,52 +1,51 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "forum_id": { - "type": ["null", "integer"] - }, - "hits": { - "type": ["null", "integer"] - }, - "id": { - "type": ["null", "integer"] - }, - "locked": { - "type": ["null", "boolean"] - }, - "merged_topic_id": { - "type": ["null", "integer"] - }, - "message": { - "type": ["null", "string"] - }, - "posts_count": { - "type": ["null", "integer"] - }, - "replied_by": { - "type": ["null", "integer"] - }, - "stamp_type": { - "type": ["null", "integer"] - }, - "sticky": { - "type": ["null", "boolean"] - }, - "title": { - "type": ["null", "string"] - }, - "user_id": { - "type": ["null", "integer"] - }, - "user_votes": { - "type": ["null", "integer"] - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "forum_id": { + "type": ["null", "integer"] + }, + "hits": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "locked": { + "type": ["null", "boolean"] + }, + "merged_topic_id": { + "type": ["null", "integer"] + }, + "message": { + "type": ["null", "string"] + }, + "posts_count": { + "type": ["null", "integer"] + }, + "replied_by": { + "type": ["null", "integer"] + }, + "stamp_type": { + "type": ["null", "integer"] + }, + "sticky": { + "type": ["null", "boolean"] + }, + "title": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "user_votes": { + "type": ["null", "integer"] + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" } } - \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/email_configs.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/email_configs.json index 2cbb675a52e93..33552923e290e 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/email_configs.json +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/email_configs.json @@ -1,37 +1,36 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "active": { - "type": ["null", "boolean"] - }, - "group_id": { - "type": ["null", "integer"] - }, - "id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "primary_role": { - "type": ["null", "boolean"] - }, - "product_id": { - "type": ["null", "integer"] - }, - "reply_email": { - "type": ["null", "string"] - }, - "to_email": { - "type": ["null", "string"] - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "active": { + "type": ["null", "boolean"] + }, + "group_id": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "primary_role": { + "type": ["null", "boolean"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "reply_email": { + "type": ["null", "string"] + }, + "to_email": { + "type": ["null", "string"] + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" } } - \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/email_mailboxes.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/email_mailboxes.json index 0f3d99cef6a13..1e604ff0224b6 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/email_mailboxes.json +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/email_mailboxes.json @@ -1,58 +1,57 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "name": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "support_email": { - "type": ["null", "string"] - }, - "group_id": { - "type": ["null", "integer"] - }, - "product_id": { - "type": ["null", "integer"] - }, - "default_reply_email": { - "type": ["null", "boolean"] - }, - "mailbox_type": { - "type": ["null", "string"] - }, - "custom_mailbox": { - "type": ["null", "string"] - }, - "access_type": { - "type": ["null", "string"] - }, - "incoming": { - "type": ["null", "string"] - }, - "mail_server": { - "type": ["null", "string"] - }, - "port": { - "type": ["null", "integer"] - }, - "use_ssl": { - "type": ["null", "boolean"] - }, - "delete_from_server": { - "type": ["null", "boolean"] - }, - "authentication": { - "type": ["null", "string"] - }, - "username": { - "type": ["null", "string"] - }, - "password": { - "type": ["null", "string"] - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "name": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "support_email": { + "type": ["null", "string"] + }, + "group_id": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "default_reply_email": { + "type": ["null", "boolean"] + }, + "mailbox_type": { + "type": ["null", "string"] + }, + "custom_mailbox": { + "type": ["null", "string"] + }, + "access_type": { + "type": ["null", "string"] + }, + "incoming": { + "type": ["null", "string"] + }, + "mail_server": { + "type": ["null", "string"] + }, + "port": { + "type": ["null", "integer"] + }, + "use_ssl": { + "type": ["null", "boolean"] + }, + "delete_from_server": { + "type": ["null", "boolean"] + }, + "authentication": { + "type": ["null", "string"] + }, + "username": { + "type": ["null", "string"] + }, + "password": { + "type": ["null", "string"] } } - \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/products.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/products.json index b9d2116eb1ed8..00d4b144b3019 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/products.json +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/products.json @@ -1,22 +1,21 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "description": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "description": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" } } - \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/scenario_automations.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/scenario_automations.json index daca2d491c649..fadfaf8bfbe89 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/scenario_automations.json +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/scenario_automations.json @@ -1,39 +1,38 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "description": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "actions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "description": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "actions": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } } - }, - "private": { - "type": ["null", "boolean"] - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" } + }, + "private": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" } } - \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/settings.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/settings.json index c0d0984ae484d..3e238217f9400 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/settings.json +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/settings.json @@ -1,22 +1,21 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "primary_language": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "primary_language": { + "type": ["null", "string"] + }, + "supported_languages": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "portal_languages": { + "type": ["null", "array"], + "items": { "type": ["null", "string"] - }, - "supported_languages": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "portal_languages": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } } } } - \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/sla_policies.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/sla_policies.json index c3cffad30b9f2..7ab01474ffa1e 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/sla_policies.json +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/sla_policies.json @@ -1,198 +1,197 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "active": { - "type": ["null", "boolean"] - }, - "description": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "is_default": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "position": { - "type": ["null", "integer"] - }, - "sla_target": { - "type": ["null", "object"], - "properties": { - "priority_4": { - "type": ["null", "object"], - "properties": { - "respond_within": { - "type": ["null", "integer"] - }, - "resolve_within": { - "type": ["null", "integer"] - }, - "business_hours": { - "type": ["null", "boolean"] - }, - "escalation_enabled": { - "type": ["null", "boolean"] - } - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "active": { + "type": ["null", "boolean"] + }, + "description": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "is_default": { + "type": ["null", "boolean"] + }, + "name": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "sla_target": { + "type": ["null", "object"], + "properties": { + "priority_4": { + "type": ["null", "object"], + "properties": { + "respond_within": { + "type": ["null", "integer"] }, - "priority_3": { - "type": ["null", "object"], - "properties": { - "respond_within": { - "type": ["null", "integer"] - }, - "resolve_within": { - "type": ["null", "integer"] - }, - "business_hours": { - "type": ["null", "boolean"] - }, - "escalation_enabled": { - "type": ["null", "boolean"] - } - } + "resolve_within": { + "type": ["null", "integer"] }, - "priority_2": { - "type": ["null", "object"], - "properties": { - "respond_within": { - "type": ["null", "integer"] - }, - "resolve_within": { - "type": ["null", "integer"] - }, - "business_hours": { - "type": ["null", "boolean"] - }, - "escalation_enabled": { - "type": ["null", "boolean"] - } - } + "business_hours": { + "type": ["null", "boolean"] }, - "priority_1": { - "type": ["null", "object"], - "properties": { - "respond_within": { - "type": ["null", "integer"] - }, - "resolve_within": { - "type": ["null", "integer"] - }, - "business_hours": { - "type": ["null", "boolean"] - }, - "escalation_enabled": { - "type": ["null", "boolean"] - } - } + "escalation_enabled": { + "type": ["null", "boolean"] } - } - }, - "applicable_to": { - "type": ["null", "object"], - "properties": { - "company_ids": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } + } + }, + "priority_3": { + "type": ["null", "object"], + "properties": { + "respond_within": { + "type": ["null", "integer"] }, - "group_ids": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } + "resolve_within": { + "type": ["null", "integer"] }, - "sources": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } + "business_hours": { + "type": ["null", "boolean"] }, - "ticket_types": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } + "escalation_enabled": { + "type": ["null", "boolean"] + } + } + }, + "priority_2": { + "type": ["null", "object"], + "properties": { + "respond_within": { + "type": ["null", "integer"] }, - "product_ids": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } + "resolve_within": { + "type": ["null", "integer"] + }, + "business_hours": { + "type": ["null", "boolean"] + }, + "escalation_enabled": { + "type": ["null", "boolean"] } + } + }, + "priority_1": { + "type": ["null", "object"], + "properties": { + "respond_within": { + "type": ["null", "integer"] + }, + "resolve_within": { + "type": ["null", "integer"] + }, + "business_hours": { + "type": ["null", "boolean"] + }, + "escalation_enabled": { + "type": ["null", "boolean"] + } + } } - }, - "escalation": { - "type": ["null", "object"], - "properties": { - "response": { - "type": ["null", "object"], - "properties": { - "escalation_time": { - "type": ["null", "integer"] - }, - "agent_ids": { - "type": ["null", "integer"] - } + } + }, + "applicable_to": { + "type": ["null", "object"], + "properties": { + "company_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "group_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "sources": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "ticket_types": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "product_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + } + } + }, + "escalation": { + "type": ["null", "object"], + "properties": { + "response": { + "type": ["null", "object"], + "properties": { + "escalation_time": { + "type": ["null", "integer"] + }, + "agent_ids": { + "type": ["null", "integer"] + } + } + }, + "resolution": { + "type": ["null", "object"], + "properties": { + "level1": { + "type": ["null", "object"], + "properties": { + "escalation_time": { + "type": ["null", "integer"] + }, + "agent_ids": { + "type": ["null", "integer"] + } + } + }, + "level2": { + "type": ["null", "object"], + "properties": { + "escalation_time": { + "type": ["null", "integer"] + }, + "agent_ids": { + "type": ["null", "integer"] + } + } + }, + "level3": { + "type": ["null", "object"], + "properties": { + "escalation_time": { + "type": ["null", "integer"] + }, + "agent_ids": { + "type": ["null", "integer"] } + } }, - "resolution": { - "type": ["null", "object"], - "properties": { - "level1": { - "type": ["null", "object"], - "properties": { - "escalation_time": { - "type": ["null", "integer"] - }, - "agent_ids": { - "type": ["null", "integer"] - } - } - }, - "level2": { - "type": ["null", "object"], - "properties": { - "escalation_time": { - "type": ["null", "integer"] - }, - "agent_ids": { - "type": ["null", "integer"] - } - } - }, - "level3": { - "type": ["null", "object"], - "properties": { - "escalation_time": { - "type": ["null", "integer"] - }, - "agent_ids": { - "type": ["null", "integer"] - } - } - }, - "level4": { - "type": ["null", "object"], - "properties": { - "escalation_time": { - "type": ["null", "integer"] - }, - "agent_ids": { - "type": ["null", "integer"] - } - } - } + "level4": { + "type": ["null", "object"], + "properties": { + "escalation_time": { + "type": ["null", "integer"] + }, + "agent_ids": { + "type": ["null", "integer"] } + } } + } } } } } - \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_articles.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_articles.json index 20f510dd62c11..766c54d20a77d 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_articles.json +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_articles.json @@ -1,69 +1,68 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "integer"] - }, - "agent_id": { - "type": ["null", "integer"] - }, - "category_id": { - "type": ["null", "integer"] - }, - "description": { - "type": ["null", "string"] - }, - "description_text": { - "type": ["null", "string"] - }, - "folder_id": { - "type": ["null", "integer"] - }, - "hits": { - "type": ["null", "integer"] - }, - "status": { - "type": ["null", "integer"] - }, - "seo_data": { - "type": ["null", "object"], - "properties": { - "meta_title": { - "type": ["null", "string"] - }, - "meta_description": { - "type": ["null", "string"] - }, - "meta_keywords": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - } - } - }, - "tags": { - "type": ["null", "array"], - "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "agent_id": { + "type": ["null", "integer"] + }, + "category_id": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + }, + "description_text": { + "type": ["null", "string"] + }, + "folder_id": { + "type": ["null", "integer"] + }, + "hits": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "integer"] + }, + "seo_data": { + "type": ["null", "object"], + "properties": { + "meta_title": { + "type": ["null", "string"] + }, + "meta_description": { + "type": ["null", "string"] + }, + "meta_keywords": { + "type": ["null", "array"], + "items": { "type": ["null", "string"] + } } - }, - "thumbs_down": { - "type": ["null", "integer"] - }, - "thumbs_up": { - "type": ["null", "integer"] - }, - "title": { + } + }, + "tags": { + "type": ["null", "array"], + "items": { "type": ["null", "string"] - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" } + }, + "thumbs_down": { + "type": ["null", "integer"] + }, + "thumbs_up": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" } } - \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_categories.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_categories.json index 1e764a4f2a408..0b578fed4a97b 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_categories.json +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_categories.json @@ -1,28 +1,27 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "description": { - "type": ["null", "string"] - }, - "id": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "description": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "visible_in_portals": { + "type": ["null", "array"], + "items": { "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "visible_in_portals": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" } + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" } } - \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_folders.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_folders.json index 810231e883713..dcf9fd54417fd 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_folders.json +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/solution_folders.json @@ -1,43 +1,42 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "description": { - "type": ["null", "string"] - }, - "id": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "description": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "visibility": { + "type": ["null", "integer"] + }, + "company_ids": { + "type": ["null", "array"], + "items": { "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "visibility": { + } + }, + "contact_segment_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "company_segment_ids": { + "type": ["null", "array"], + "items": { "type": ["null", "integer"] - }, - "company_ids": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } - }, - "contact_segment_ids": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } - }, - "company_segment_ids": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" } + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" } } - \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/ticket_fields.json b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/ticket_fields.json index ae302f14b7a55..e84ee5d46547c 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/ticket_fields.json +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/schemas/ticket_fields.json @@ -1,95 +1,94 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "integer"] - }, - "default": { - "type": ["null", "boolean"] - }, - "description": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "position": { - "type": ["null", "integer"] - }, - "required_for_closure": { - "type": ["null", "boolean"] - }, - "type": { - "type": ["null", "string"] - }, - "required_for_agents": { - "type": ["null", "boolean"] - }, - "required_for_customers": { - "type": ["null", "boolean"] - }, - "label_for_customers": { - "type": ["null", "string"] - }, - "customers_can_edit": { - "type": ["null", "boolean"] - }, - "displayed_to_customers": { - "type": ["null", "boolean"] - }, - "portal_cc": { - "type": ["null", "boolean"] - }, - "portal_cc_to": { - "type": ["null", "string"] - }, - "choices": { - "type": ["null", "array", "object"] - }, - "is_fsm": { - "type": ["null", "boolean"] - }, - "field_update_in_progress": { - "type": ["null", "boolean"] - }, - "dependent_fields": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "string"] - }, - "label_for_customers": { - "type": ["null", "string"] - }, - "level": { - "type": ["null", "integer"] - }, - "ticket_field_id": { - "type": ["null", "integer"] - }, - "created_at": { - "type": ["null", "string"] - }, - "updated_at": { - "type": ["null", "string"] - } - } - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "description": { + "type": ["null", "string"] + }, + "label": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "required_for_closure": { + "type": ["null", "boolean"] + }, + "type": { + "type": ["null", "string"] + }, + "required_for_agents": { + "type": ["null", "boolean"] + }, + "required_for_customers": { + "type": ["null", "boolean"] + }, + "label_for_customers": { + "type": ["null", "string"] + }, + "customers_can_edit": { + "type": ["null", "boolean"] + }, + "displayed_to_customers": { + "type": ["null", "boolean"] + }, + "portal_cc": { + "type": ["null", "boolean"] + }, + "portal_cc_to": { + "type": ["null", "string"] + }, + "choices": { + "type": ["null", "array", "object"] + }, + "is_fsm": { + "type": ["null", "boolean"] + }, + "field_update_in_progress": { + "type": ["null", "boolean"] + }, + "dependent_fields": { + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "label": { + "type": ["null", "string"] + }, + "label_for_customers": { + "type": ["null", "string"] + }, + "level": { + "type": ["null", "integer"] + }, + "ticket_field_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + } } + }, + "created_at": { + "type": "string" + }, + "updated_at": { + "type": "string" } } - \ No newline at end of file +} From 7b669f423791f4051efd508619a362cf06625596 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 3 Jun 2022 14:30:44 +0200 Subject: [PATCH 17/18] bump version --- airbyte-integrations/connectors/source-freshdesk/Dockerfile | 2 +- docs/integrations/sources/freshdesk.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-freshdesk/Dockerfile b/airbyte-integrations/connectors/source-freshdesk/Dockerfile index da9cf238c8bbe..436fdcb98061f 100644 --- a/airbyte-integrations/connectors/source-freshdesk/Dockerfile +++ b/airbyte-integrations/connectors/source-freshdesk/Dockerfile @@ -34,5 +34,5 @@ COPY source_freshdesk ./source_freshdesk ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.0 +LABEL io.airbyte.version=0.3.1 LABEL io.airbyte.name=airbyte/source-freshdesk diff --git a/docs/integrations/sources/freshdesk.md b/docs/integrations/sources/freshdesk.md index 6a63af9f4c8c9..89915abf8b176 100644 --- a/docs/integrations/sources/freshdesk.md +++ b/docs/integrations/sources/freshdesk.md @@ -53,6 +53,7 @@ Please read [How to find your API key](https://support.freshdesk.com/support/sol | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------| +| 0.3.1 | 2022-06-03 | [13332](https://github.com/airbytehq/airbyte/pull/13332) | Add new streams | 0.3.0 | 2022-05-30 | [12334](https://github.com/airbytehq/airbyte/pull/12334) | Implement with latest CDK | 0.2.11 | 2021-12-14 | [8682](https://github.com/airbytehq/airbyte/pull/8682) | Migrate to the CDK | | 0.2.10 | 2021-12-06 | [8524](https://github.com/airbytehq/airbyte/pull/8524) | Update connector fields title/description | From e68c6d95b8b1e76b93b98314ef2a9d2f38e40410 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 3 Jun 2022 12:54:49 +0000 Subject: [PATCH 18/18] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 8c9ac446e5502..5c2ecdb147300 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -271,7 +271,7 @@ - name: Freshdesk sourceDefinitionId: ec4b9503-13cb-48ab-a4ab-6ade4be46567 dockerRepository: airbyte/source-freshdesk - dockerImageTag: 0.3.0 + dockerImageTag: 0.3.1 documentationUrl: https://docs.airbyte.io/integrations/sources/freshdesk icon: freshdesk.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index fbccfd14538cf..55d9a6599b1e0 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -2489,7 +2489,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-freshdesk:0.3.0" +- dockerImage: "airbyte/source-freshdesk:0.3.1" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/freshdesk" connectionSpecification: