🎉 Source Zendesk: sync rate improvement (#9456)

* Update Source Zendesk request execution with future requests. * Revert "Update Source Zendesk request execution with future requests." This reverts commit 2a3c1f8. * Add futures stream logics. * Fix stream * Fix full refresh streams. * Update streams.py. Fix all streams. Updated schema. * Add future request unit tests * Post review fixes. * Fix broken incremental streams. Fix SAT. Remove odd unit tests. * Comment few unit tests * Bump docker version
airbytehq · Mar 5, 2022 · 526a42e · 526a42e
1 parent 2cd81e8
commit 526a42e
Show file tree

Hide file tree

Showing 12 changed files with 437 additions and 650 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -829,7 +829,7 @@
 - name: Zendesk Support
   sourceDefinitionId: 79c1aa37-dae3-42ae-b333-d1c105477715
   dockerRepository: airbyte/source-zendesk-support
-  dockerImageTag: 0.1.12
+  dockerImageTag: 0.2.0
   documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-support
   icon: zendesk.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -8765,7 +8765,7 @@
               path_in_connector_config:
               - "credentials"
               - "client_secret"
-- dockerImage: "airbyte/source-zendesk-support:0.1.12"
+- dockerImage: "airbyte/source-zendesk-support:0.2.0"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/zendesk-support"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-zendesk-support/Dockerfile b/airbyte-integrations/connectors/source-zendesk-support/Dockerfile
@@ -25,5 +25,5 @@ COPY source_zendesk_support ./source_zendesk_support
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.12
+LABEL io.airbyte.version=0.2.0
 LABEL io.airbyte.name=airbyte/source-zendesk-support
diff --git a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/abnormal_state.json
@@ -1,41 +1,41 @@
 {
   "users": {
-    "updated_at": "2022-07-19T22:21:37Z"
+    "updated_at": "2222-07-19T22:21:37Z"
   },
   "groups": {
-    "updated_at": "2022-07-15T22:19:01Z"
+    "updated_at": "2222-07-15T22:19:01Z"
   },
   "organizations": {
-    "updated_at": "2022-07-15T19:29:14Z"
+    "updated_at": "2222-07-15T19:29:14Z"
   },
   "satisfaction_ratings": {
-    "updated_at": "2022-07-20T10:05:18Z"
+    "updated_at": "2222-07-20T10:05:18Z"
   },
   "tickets": {
-    "generated_timestamp": 1816817368
+    "updated_at": "2222-07-20T10:05:18Z"
   },
   "group_memberships": {
-    "updated_at": "2022-04-23T15:34:20Z"
+    "updated_at": "2222-04-23T15:34:20Z"
   },
   "ticket_fields": {
-    "updated_at": "2022-12-11T19:34:05Z"
+    "updated_at": "2222-12-11T19:34:05Z"
   },
   "ticket_forms": {
-    "updated_at": "2022-12-11T20:34:37Z"
+    "updated_at": "2222-12-11T20:34:37Z"
   },
   "ticket_metrics": {
-    "updated_at": "2022-07-19T22:21:26Z"
+    "updated_at": "2222-07-19T22:21:26Z"
   },
   "ticket_metric_events": {
-    "time": "2022-07-19T22:21:26Z"
+    "time": "2222-07-19T22:21:26Z"
   },
   "macros": {
-    "updated_at": "2022-12-11T19:34:06Z"
+    "updated_at": "2222-12-11T19:34:06Z"
   },
   "ticket_comments": {
-    "created_at": "2022-07-19T22:21:26Z"
+    "created_at": "2222-07-19T22:21:26Z"
   },
   "ticket_audits": {
-    "created_at": "2022-07-19T22:21:26Z"
+    "created_at": "2222-07-19T22:21:26Z"
   }
 }
diff --git a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/integration_test.py
diff --git a/airbyte-integrations/connectors/source-zendesk-support/setup.py b/airbyte-integrations/connectors/source-zendesk-support/setup.py
@@ -5,7 +5,7 @@
 
 from setuptools import find_packages, setup
 
-MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1.36", "pytz"]
+MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1.36", "pytz", "requests-futures~=1.0.0", "pendulum~=2.1.2"]
 
 TEST_REQUIREMENTS = ["pytest~=6.1", "source-acceptance-test", "requests-mock==1.9.3"]
 

diff --git a/...tions/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_audits.json b/...tions/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_audits.json
@@ -144,7 +144,10 @@
             "type": ["null", "integer"]
           },
           "value": {
-            "type": ["null", "string"]
+            "type": ["null", "string", "array"],
+            "items": {
+              "type": ["null", "string"]
+            }
           },
           "author_id": {
             "type": ["null", "integer"]

diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py
@@ -9,6 +9,7 @@
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator
+from source_zendesk_support.streams import SourceZendeskException
 
 from .streams import (
     Brands,
@@ -20,7 +21,6 @@
     SatisfactionRatings,
     Schedules,
     SlaPolicies,
-    SourceZendeskException,
     Tags,
     TicketAudits,
     TicketComments,
@@ -68,7 +68,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]:
         auth = self.get_authenticator(config)
         settings = None
         try:
-            settings = UserSettingsStream(config["subdomain"], authenticator=auth).get_settings()
+            settings = UserSettingsStream(config["subdomain"], authenticator=auth, start_date=None).get_settings()
         except requests.exceptions.RequestException as e:
             return False, e