🎉 Source SurveyMonkey - to beta (#14998)

* fixed incremental syns for response stream, added unittest, fixed specs, fixed incremental SAT * removed comments * updated docs * updated docs * bumped connector version * bumped release stage * auto-bump connector version [ci skip] * updated source_specs.yaml Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Aug 3, 2022 · 8fe89f5 · 8fe89f5
1 parent e2526ff
commit 8fe89f5
Show file tree

Hide file tree

Showing 18 changed files with 768 additions and 183 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -945,11 +945,11 @@
 - name: SurveyMonkey
   sourceDefinitionId: badc5925-0485-42be-8caa-b34096cb71b5
   dockerRepository: airbyte/source-surveymonkey
-  dockerImageTag: 0.1.8
+  dockerImageTag: 0.1.9
   documentationUrl: https://docs.airbyte.io/integrations/sources/surveymonkey
   icon: surveymonkey.svg
   sourceType: api
-  releaseStage: alpha
+  releaseStage: beta
 - name: TalkDesk Explore
   sourceDefinitionId: f00d2cf4-3c28-499a-ba93-b50b6f26359e
   dockerRepository: airbyte/source-talkdesk-explore

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -9367,34 +9367,38 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-surveymonkey:0.1.8"
+- dockerImage: "airbyte/source-surveymonkey:0.1.9"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/surveymonkey"
     connectionSpecification:
       $schema: "http://json-schema.org/draft-07/schema#"
       title: "SurveyMonkey Spec"
       type: "object"
       required:
+      - "access_token"
       - "start_date"
       additionalProperties: true
       properties:
+        access_token:
+          title: "Access Token"
+          order: 0
+          type: "string"
+          airbyte_secret: true
+          description: "Access Token for making authenticated requests. See the <a\
+            \ href=\"https://docs.airbyte.io/integrations/sources/surveymonkey\">docs</a>\
+            \ for information on how to generate this key."
         start_date:
           title: "Start Date"
+          order: 1
           type: "string"
           description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\
             \ data before this date will not be replicated."
           pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z?$"
           examples:
           - "2021-01-01T00:00:00Z"
-        access_token:
-          title: "Access Token"
-          type: "string"
-          airbyte_secret: true
-          description: "Access Token for making authenticated requests. See the <a\
-            \ href=\"https://docs.airbyte.io/integrations/sources/surveymonkey\">docs</a>\
-            \ for information on how to generate this key."
         survey_ids:
           type: "array"
+          order: 2
           items:
             type: "string"
             pattern: "^[0-9]{8,9}$"

diff --git a/airbyte-integrations/connectors/source-surveymonkey/Dockerfile b/airbyte-integrations/connectors/source-surveymonkey/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.8
+LABEL io.airbyte.version=0.1.9
 LABEL io.airbyte.name=airbyte/source-surveymonkey
diff --git a/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-config.yml b/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-config.yml
@@ -12,13 +12,9 @@ tests:
   basic_read:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
-  # We cannot use these tests for testing Incremental,
-  # since for Surveymonkey the State is saved for each Survey separately,
-  # and the Acceptance Tests at this stage do not support this functionality.
-  #incremental:
-  #  - config_path: "secrets/config.json"
-  #    configured_catalog_path: "integration_tests/configured_catalog.json"
-  #    future_state_path: "integration_tests/abnormal_state.json"
+  incremental:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
   full_refresh:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
diff --git a/airbyte-integrations/connectors/source-surveymonkey/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/abnormal_state.json
@@ -0,0 +1,46 @@
+{
+  "surveys": {
+    "date_modified": "2022-06-10T11:02:01"
+  },
+  "survey_responses": {
+    "306079584": {
+      "date_modified": "2022-06-08T18:17:18+00:00"
+    },
+    "307785429": {
+      "date_modified": "2022-06-10T10:59:43+00:00"
+    },
+    "307785444": {
+      "date_modified": "2022-06-10T11:00:19+00:00"
+    },
+    "307785394": {
+      "date_modified": "2022-06-10T11:00:59+00:00"
+    },
+    "307785402": {
+      "date_modified": "2022-06-10T11:01:31+00:00"
+    },
+    "307785408": {
+      "date_modified": "2022-06-10T11:02:08+00:00"
+    },
+    "307785448": {
+      "date_modified": "2022-06-10T11:02:49+00:00"
+    },
+    "307784834": {
+      "date_modified": "2022-06-10T11:03:45+00:00"
+    },
+    "307784863": {
+      "date_modified": "2022-06-10T11:04:29+00:00"
+    },
+    "307784846": {
+      "date_modified": "2022-06-10T11:05:05+00:00"
+    },
+    "307784856": {
+      "date_modified": "2022-06-10T11:05:44+00:00"
+    },
+    "307785388": {
+      "date_modified": "2022-06-10T11:06:20+00:00"
+    },
+    "307785415": {
+      "date_modified": "2022-06-10T11:06:43+00:00"
+    }
+  }
+}
diff --git a/airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_ids.json b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_ids.json
@@ -0,0 +1,13 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "survey_ids",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "append"
+    }
+  ]
+}
diff --git a/airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_pages.json b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_pages.json
@@ -0,0 +1,16 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "survey_pages",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["date_modified"]
+      },
+      "sync_mode": "full_refresh",
+      "cursor_field": ["date_modified"],
+      "destination_sync_mode": "append"
+    }
+  ]
+}
diff --git a/airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_questions.json b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_questions.json
@@ -0,0 +1,16 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "survey_questions",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["date_modified"]
+      },
+      "sync_mode": "full_refresh",
+      "cursor_field": ["date_modified"],
+      "destination_sync_mode": "append"
+    }
+  ]
+}
diff --git a/airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_responses.json b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_responses.json
@@ -0,0 +1,16 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "survey_responses",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["date_modified"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["date_modified"],
+      "destination_sync_mode": "append"
+    }
+  ]
+}
diff --git a/airbyte-integrations/connectors/source-surveymonkey/integration_tests/surveys.json b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/surveys.json
@@ -0,0 +1,16 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "surveys",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["date_modified"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["date_modified"],
+      "destination_sync_mode": "append"
+    }
+  ]
+}
diff --git a/airbyte-integrations/connectors/source-surveymonkey/sample_files/state.json b/airbyte-integrations/connectors/source-surveymonkey/sample_files/state.json
@@ -1,10 +1,48 @@
 {
   "surveys": {
-    "date_modified": "2021-06-10T11:02:01"
+    "date_modified": "2021-06-10T11:07:00"
   },
   "survey_responses": {
+    "306079584": {
+      "date_modified": "2021-06-08T18:17:18+00:00"
+    },
+    "307785429": {
+      "date_modified": "2021-06-10T10:59:43+00:00"
+    },
+    "307785444": {
+      "date_modified": "2021-06-10T11:00:19+00:00"
+    },
+    "307785394": {
+      "date_modified": "2021-06-10T11:00:59+00:00"
+    },
+    "307785402": {
+      "date_modified": "2021-06-10T11:01:31+00:00"
+    },
+    "307785408": {
+      "date_modified": "2021-06-10T11:02:08+00:00"
+    },
     "307785448": {
       "date_modified": "2021-06-10T11:02:49+00:00"
+    },
+    "307784834": {
+      "date_modified": "2021-06-10T11:03:45+00:00"
+    },
+    "307784863": {
+      "date_modified": "2021-06-10T11:04:29+00:00"
+    },
+    "307784846": {
+      "date_modified": "2021-06-10T11:05:05+00:00"
+    },
+    "307784856": {
+      "date_modified": "2021-06-10T11:05:44+00:00"
+    },
+    "307785388": {
+      "date_modified": "2021-06-10T11:06:20+00:00"
+    },
+    "307785415": {
+      "date_modified": "2021-06-10T11:06:43+00:00"
     }
   }
+
+
 }
diff --git a/airbyte-integrations/connectors/source-surveymonkey/setup.py b/airbyte-integrations/connectors/source-surveymonkey/setup.py
@@ -7,10 +7,7 @@
 
 MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "vcrpy==4.1.1"]
 
-TEST_REQUIREMENTS = [
-    "pytest~=6.1",
-    "source-acceptance-test",
-]
+TEST_REQUIREMENTS = ["pytest~=6.1", "source-acceptance-test", "requests_mock"]
 
 setup(
     name="source_surveymonkey",

diff --git a/...e-integrations/connectors/source-surveymonkey/source_surveymonkey/schemas/survey_ids.json b/...e-integrations/connectors/source-surveymonkey/source_surveymonkey/schemas/survey_ids.json
@@ -0,0 +1,18 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "id": {
+      "type": ["string", "null"]
+    },
+    "title": {
+      "type": ["string", "null"]
+    },
+    "nickname": {
+      "type": ["string", "null"]
+    },
+    "href": {
+      "type": ["string", "null"]
+    }
+  }
+}
diff --git a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/spec.json b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/spec.json
@@ -4,24 +4,27 @@
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "SurveyMonkey Spec",
     "type": "object",
-    "required": ["start_date"],
+    "required": ["access_token", "start_date"],
     "additionalProperties": true,
     "properties": {
+      "access_token": {
+        "title": "Access Token",
+        "order": 0,
+        "type": "string",
+        "airbyte_secret": true,
+        "description": "Access Token for making authenticated requests. See the <a href=\"https://docs.airbyte.io/integrations/sources/surveymonkey\">docs</a> for information on how to generate this key."
+      },
       "start_date": {
         "title": "Start Date",
+        "order": 1,
         "type": "string",
         "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.",
         "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z?$",
         "examples": ["2021-01-01T00:00:00Z"]
       },
-      "access_token": {
-        "title": "Access Token",
-        "type": "string",
-        "airbyte_secret": true,
-        "description": "Access Token for making authenticated requests. See the <a href=\"https://docs.airbyte.io/integrations/sources/surveymonkey\">docs</a> for information on how to generate this key."
-      },
       "survey_ids": {
         "type": "array",
+        "order": 2,
         "items": {
           "type": "string",
           "pattern": "^[0-9]{8,9}$"