From 146694bd3293122e2e1d08d817f0f2c6bcc2f14b Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Mon, 25 Jul 2022 03:04:03 +0300 Subject: [PATCH 1/8] fixed incremental syns for response stream, added unittest, fixed specs, fixed incremental SAT --- .../acceptance-test-config.yml | 10 +- .../integration_tests/abnormal_state.json | 46 ++ .../integration_tests/survey_ids.json | 13 + .../integration_tests/survey_pages.json | 16 + .../integration_tests/survey_questions.json | 16 + .../integration_tests/survey_responses.json | 16 + .../integration_tests/surveys.json | 16 + .../sample_files/state.json | 40 +- .../connectors/source-surveymonkey/setup.py | 5 +- .../schemas/survey_ids.json | 18 + .../source_surveymonkey/spec.json | 17 +- .../source_surveymonkey/streams.py | 208 +++++---- .../unit_tests/test_source.py | 28 ++ .../unit_tests/test_streams.py | 414 ++++++++++++++++++ 14 files changed, 735 insertions(+), 128 deletions(-) create mode 100644 airbyte-integrations/connectors/source-surveymonkey/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_ids.json create mode 100644 airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_pages.json create mode 100644 airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_questions.json create mode 100644 airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_responses.json create mode 100644 airbyte-integrations/connectors/source-surveymonkey/integration_tests/surveys.json create mode 100644 airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/schemas/survey_ids.json create mode 100644 airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py diff --git a/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-config.yml b/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-config.yml index 8b261bbe5d84..fd2f549bb517 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-config.yml @@ -12,13 +12,9 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - # We cannot use these tests for testing Incremental, - # since for Surveymonkey the State is saved for each Survey separately, - # and the Acceptance Tests at this stage do not support this functionality. - #incremental: - # - config_path: "secrets/config.json" - # configured_catalog_path: "integration_tests/configured_catalog.json" - # future_state_path: "integration_tests/abnormal_state.json" + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-surveymonkey/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..629a07a2805e --- /dev/null +++ b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/abnormal_state.json @@ -0,0 +1,46 @@ +{ + "surveys": { + "date_modified": "2022-06-10T11:02:01" + }, + "survey_responses": { + "306079584": { + "date_modified": "2022-06-08T18:17:18+00:00" + }, + "307785429": { + "date_modified": "2022-06-10T10:59:43+00:00" + }, + "307785444": { + "date_modified": "2022-06-10T11:00:19+00:00" + }, + "307785394": { + "date_modified": "2022-06-10T11:00:59+00:00" + }, + "307785402": { + "date_modified": "2022-06-10T11:01:31+00:00" + }, + "307785408": { + "date_modified": "2022-06-10T11:02:08+00:00" + }, + "307785448": { + "date_modified": "2022-06-10T11:02:49+00:00" + }, + "307784834": { + "date_modified": "2022-06-10T11:03:45+00:00" + }, + "307784863": { + "date_modified": "2022-06-10T11:04:29+00:00" + }, + "307784846": { + "date_modified": "2022-06-10T11:05:05+00:00" + }, + "307784856": { + "date_modified": "2022-06-10T11:05:44+00:00" + }, + "307785388": { + "date_modified": "2022-06-10T11:06:20+00:00" + }, + "307785415": { + "date_modified": "2022-06-10T11:06:43+00:00" + } + } +} diff --git a/airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_ids.json b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_ids.json new file mode 100644 index 000000000000..ded81ab6e416 --- /dev/null +++ b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_ids.json @@ -0,0 +1,13 @@ +{ + "streams": [ + { + "stream": { + "name": "survey_ids", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_pages.json b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_pages.json new file mode 100644 index 000000000000..9f1c2f5eedb8 --- /dev/null +++ b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_pages.json @@ -0,0 +1,16 @@ +{ + "streams": [ + { + "stream": { + "name": "survey_pages", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date_modified"] + }, + "sync_mode": "full_refresh", + "cursor_field": ["date_modified"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_questions.json b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_questions.json new file mode 100644 index 000000000000..8cd35a4af671 --- /dev/null +++ b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_questions.json @@ -0,0 +1,16 @@ +{ + "streams": [ + { + "stream": { + "name": "survey_questions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date_modified"] + }, + "sync_mode": "full_refresh", + "cursor_field": ["date_modified"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_responses.json b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_responses.json new file mode 100644 index 000000000000..342002a0817a --- /dev/null +++ b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/survey_responses.json @@ -0,0 +1,16 @@ +{ + "streams": [ + { + "stream": { + "name": "survey_responses", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date_modified"] + }, + "sync_mode": "incremental", + "cursor_field": ["date_modified"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-surveymonkey/integration_tests/surveys.json b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/surveys.json new file mode 100644 index 000000000000..9baf821240f1 --- /dev/null +++ b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/surveys.json @@ -0,0 +1,16 @@ +{ + "streams": [ + { + "stream": { + "name": "surveys", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date_modified"] + }, + "sync_mode": "incremental", + "cursor_field": ["date_modified"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-surveymonkey/sample_files/state.json b/airbyte-integrations/connectors/source-surveymonkey/sample_files/state.json index 215763853c08..68263ca6b0fc 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/sample_files/state.json +++ b/airbyte-integrations/connectors/source-surveymonkey/sample_files/state.json @@ -1,10 +1,48 @@ { "surveys": { - "date_modified": "2021-06-10T11:02:01" + "date_modified": "2021-06-10T11:07:00" }, "survey_responses": { + "306079584": { + "date_modified": "2021-06-08T18:17:18+00:00" + }, + "307785429": { + "date_modified": "2021-06-10T10:59:43+00:00" + }, + "307785444": { + "date_modified": "2021-06-10T11:00:19+00:00" + }, + "307785394": { + "date_modified": "2021-06-10T11:00:59+00:00" + }, + "307785402": { + "date_modified": "2021-06-10T11:01:31+00:00" + }, + "307785408": { + "date_modified": "2021-06-10T11:02:08+00:00" + }, "307785448": { "date_modified": "2021-06-10T11:02:49+00:00" + }, + "307784834": { + "date_modified": "2021-06-10T11:03:45+00:00" + }, + "307784863": { + "date_modified": "2021-06-10T11:04:29+00:00" + }, + "307784846": { + "date_modified": "2021-06-10T11:05:05+00:00" + }, + "307784856": { + "date_modified": "2021-06-10T11:05:44+00:00" + }, + "307785388": { + "date_modified": "2021-06-10T11:06:20+00:00" + }, + "307785415": { + "date_modified": "2021-06-10T11:06:43+00:00" } } + + } diff --git a/airbyte-integrations/connectors/source-surveymonkey/setup.py b/airbyte-integrations/connectors/source-surveymonkey/setup.py index ca3d139ab348..39bd917b0c31 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/setup.py +++ b/airbyte-integrations/connectors/source-surveymonkey/setup.py @@ -7,10 +7,7 @@ MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "vcrpy==4.1.1"] -TEST_REQUIREMENTS = [ - "pytest~=6.1", - "source-acceptance-test", -] +TEST_REQUIREMENTS = ["pytest~=6.1", "source-acceptance-test", "requests_mock"] setup( name="source_surveymonkey", diff --git a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/schemas/survey_ids.json b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/schemas/survey_ids.json new file mode 100644 index 000000000000..4502c94c445d --- /dev/null +++ b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/schemas/survey_ids.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "title": { + "type": ["string", "null"] + }, + "nickname": { + "type": ["string", "null"] + }, + "href": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/spec.json b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/spec.json index aa734e610e73..eb697230c76d 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/spec.json +++ b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/spec.json @@ -4,24 +4,27 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "SurveyMonkey Spec", "type": "object", - "required": ["start_date"], + "required": ["access_token", "start_date"], "additionalProperties": true, "properties": { + "access_token": { + "title": "Access Token", + "order": 0, + "type": "string", + "airbyte_secret": true, + "description": "Access Token for making authenticated requests. See the docs for information on how to generate this key." + }, "start_date": { "title": "Start Date", + "order": 1, "type": "string", "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z?$", "examples": ["2021-01-01T00:00:00Z"] }, - "access_token": { - "title": "Access Token", - "type": "string", - "airbyte_secret": true, - "description": "Access Token for making authenticated requests. See the docs for information on how to generate this key." - }, "survey_ids": { "type": "array", + "order": 2, "items": { "type": "string", "pattern": "^[0-9]{8,9}$" diff --git a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py index c2b6e936bb8e..7df340300d21 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py +++ b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py @@ -20,23 +20,13 @@ class SurveymonkeyStream(HttpStream, ABC): url_base = "https://api.surveymonkey.com/v3/" primary_key = "id" data_field = "data" + default_backoff_time: int = 60 # secs def __init__(self, start_date: pendulum.datetime, survey_ids: List[str], **kwargs): super().__init__(**kwargs) self._start_date = start_date self._survey_ids = survey_ids - def backoff_time(self, response: requests.Response) -> Optional[float]: - """ - # respecting daily limit if valid - daily_limit_remaining = int(response.headers.get('X-Ratelimit-App-Global-Day-Remaining')) - if daily_limit_remaining and daily_limit_remaining<=0: - return response.headers.get('X-Ratelimit-App-Global-Day-Reset') - """ - minute_limit_remaining = int(response.headers.get("X-Ratelimit-App-Global-Minute-Remaining")) - if minute_limit_remaining and minute_limit_remaining <= 1: - return 60 - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: resp_json = response.json() links = resp_json.get("links", {}) @@ -45,29 +35,14 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, params = dict(urllib.parse.parse_qsl(next_query_string)) return params - def request_headers(self, **kwargs) -> Mapping[str, Any]: - return {"Content-Type": "application/json"} - - def raise_error_from_response(self, response_json): - """ - this method use in all parse responses - including those who does not inherit / super() due to - necessity use raw response instead of accessing `data_field` - """ - if response_json.get("error"): - raise Exception(repr(response_json.get("error"))) - - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: - response_json = response.json() - self.raise_error_from_response(response_json=response_json) - result = response_json.get(self.data_field, []) - yield from result - def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: return next_page_token or {} + def request_headers(self, **kwargs) -> Mapping[str, Any]: + return {"Content-Type": "application/json"} + def read_records( self, sync_mode: SyncMode, @@ -85,6 +60,43 @@ def read_records( sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state ) + def backoff_time(self, response: requests.Response) -> Optional[float]: + """ + https://developer.surveymonkey.com/api/v3/#headers + X-Ratelimit-App-Global-Minute-Remaining - Number of remaining requests app has before hitting per minute limit + X-Ratelimit-App-Global-Minute-Reset - Number of seconds until the rate limit remaining resets + + Limits: https://developer.surveymonkey.com/api/v3/#request-and-response-limits + Max Requests Per Day - 500 + Max Requests Per Minute - 120 + + Real limits from API response headers: + "X-Ratelimit-App-Global-Minute-Limit": "720" + "X-Ratelimit-App-Global-Day-Limit": "500000" + """ + # Stop for 60 secs if less than 1 request remains before we hit minute limit + minute_limit_remaining = response.headers.get("X-Ratelimit-App-Global-Minute-Remaining", "100") + if int(minute_limit_remaining) <= 1: + return self.default_backoff_time + + def raise_error_from_response(self, response_json): + """ + this method use in all parse responses + including those who does not inherit / super() due to + necessity use raw response instead of accessing `data_field` + """ + if response_json.get("error"): + raise Exception(repr(response_json.get("error"))) + + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + response_json = response.json() + self.raise_error_from_response(response_json=response_json) + + if self.data_field: + yield from response_json.get(self.data_field, []) + else: + yield response_json + class IncrementalSurveymonkeyStream(SurveymonkeyStream, ABC): @@ -93,10 +105,23 @@ class IncrementalSurveymonkeyStream(SurveymonkeyStream, ABC): @property @abstractmethod def cursor_field(self) -> str: + pass + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: """ - Defining a cursor field indicates that a stream is incremental, so any incremental stream must extend this class - and define a cursor field. + Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object + and returning an updated state object. """ + state_value = max(current_stream_state.get(self.cursor_field, ""), latest_record.get(self.cursor_field, "")) + return {self.cursor_field: state_value} + + +class SurveyIds(IncrementalSurveymonkeyStream): + + cursor_field = "date_modified" + + def path(self, **kwargs) -> str: + return "surveys" def request_params(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: params = super().request_params(stream_state=stream_state, **kwargs) @@ -109,52 +134,33 @@ def request_params(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMa params["start_modified_at"] = since_value.strftime("%Y-%m-%dT%H:%M:%S") return params - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object - and returning an updated state object. - """ - return {self.cursor_field: max(latest_record.get(self.cursor_field), current_stream_state.get(self.cursor_field, ""))} +class SliceMixin: + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"surveys/{stream_slice['survey_id']}/details" + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs): + if self._survey_ids: + yield from [{"survey_id": id} for id in self._survey_ids] + else: + survey_stream = SurveyIds(start_date=self._start_date, survey_ids=self._survey_ids, authenticator=self.authenticator) + for survey in survey_stream.read_records(sync_mode=SyncMode.full_refresh, stream_state=stream_state): + yield {"survey_id": survey["id"]} -class Surveys(IncrementalSurveymonkeyStream): + +class Surveys(SliceMixin, IncrementalSurveymonkeyStream): """ Docs: https://developer.surveymonkey.com/api/v3/#surveys A source for stream slices. It does not contain useful info itself. - """ - cursor_field = "date_modified" - - def path(self, **kwargs) -> str: - return "surveys" - - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: - # params = super().request_params(stream_state=stream_state, **kwargs) - survey_ids = self._survey_ids - result = super().parse_response(response=response, stream_state=stream_state, **kwargs) - for record in result: - substream = SurveyDetails( - survey_id=record["id"], start_date=self._start_date, survey_ids=survey_ids, authenticator=self.authenticator - ) - child_record = substream.read_records(sync_mode=SyncMode.full_refresh) - if not survey_ids or record["id"] in survey_ids: - substream = SurveyDetails( - survey_id=record["id"], start_date=self._start_date, survey_ids=survey_ids, authenticator=self.authenticator - ) - child_record = substream.read_records(sync_mode=SyncMode.full_refresh) - yield from child_record - - -class SurveyDetails(SurveymonkeyStream): - """ - The `/id/details` endpoint contains full data about pages and questions. This data is already collected and + The `surveys/id/details` endpoint contains full data about pages and questions. This data is already collected and gathered into array [pages] and array of arrays questions, where each inner array contains data about certain page. Example [[q1, q2,q3], [q4,q5]] means we have 2 pages, first page contains 3 questions q1, q2, q3, second page contains other. If we use the "normal" query, we need to query surveys/id/pages for page enumeration, - then we need to query each page_id in every new request for details (because `pages` doesn't contain full info - and valid only for enumeration), then for each page need to enumerate questions and get each question_id for details - (since `/surveys/id/pages/id/questions` without ending /id also doesnt contain full info, + then we need to query each page_id in every new request for details (because `pages` doesn't contain full info + and valid only for enumeration), then for each page need to enumerate questions and get each question_id for details + (since `/surveys/id/pages/id/questions` without ending /id also doesnt contain full info, In other words, we need to have triple stream slices, (note that api is very very rate limited and we need details for each survey etc), and finally we get a response similar to those we can have from `/id/details` @@ -163,55 +169,33 @@ class SurveyDetails(SurveymonkeyStream): So this way is very much better in terms of API limits. """ - def __init__(self, survey_id, start_date, survey_ids, **kwargs): - self.survey_id = survey_id - super().__init__(start_date=start_date, survey_ids=survey_ids, **kwargs) - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return f"surveys/{self.survey_id}/details" + data_field = None + cursor_field = "date_modified" def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: - response_json = response.json() - self.raise_error_from_response(response_json=response_json) - response_json.pop("pages", None) - yield response_json + data = super().parse_response(response=response, stream_state=stream_state, **kwargs) + for record in data: + record.pop("pages", None) # remove pages data + yield record -class SurveyPages(SurveymonkeyStream): +class SurveyPages(SliceMixin, SurveymonkeyStream): """should be filled from SurveyDetails""" data_field = "pages" - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - survey_id = stream_slice["survey_id"] - return f"surveys/{survey_id}/details" - - def stream_slices(self, **kwargs): - survey_stream = Surveys(start_date=self._start_date, survey_ids=self._survey_ids, authenticator=self.authenticator) - for survey in survey_stream.read_records(sync_mode=SyncMode.full_refresh): - yield {"survey_id": survey["id"]} - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: data = super().parse_response(response=response, stream_state=stream_state, **kwargs) for record in data: - record.pop("questions", None) + record.pop("questions", None) # remove question data yield record -class SurveyQuestions(SurveymonkeyStream): +class SurveyQuestions(SliceMixin, SurveymonkeyStream): """should be filled from SurveyDetails""" data_field = "pages" - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - survey_id = stream_slice["survey_id"] - return f"surveys/{survey_id}/details" - - def stream_slices(self, **kwargs): - survey_stream = Surveys(start_date=self._start_date, survey_ids=self._survey_ids, authenticator=self.authenticator) - for survey in survey_stream.read_records(sync_mode=SyncMode.full_refresh): - yield {"survey_id": survey["id"]} - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: data = super().parse_response(response=response, stream_state=stream_state, **kwargs) for entry in data: @@ -222,7 +206,7 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, yield question -class SurveyResponses(IncrementalSurveymonkeyStream): +class SurveyResponses(SliceMixin, IncrementalSurveymonkeyStream): """ Docs: https://developer.surveymonkey.com/api/v3/#survey-responses """ @@ -230,13 +214,7 @@ class SurveyResponses(IncrementalSurveymonkeyStream): cursor_field = "date_modified" def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - survey_id = stream_slice["survey_id"] - return f"surveys/{survey_id}/responses/bulk" - - def stream_slices(self, **kwargs): - survey_stream = Surveys(start_date=self._start_date, survey_ids=self._survey_ids, authenticator=self.authenticator) - for survey in survey_stream.read_records(sync_mode=SyncMode.full_refresh): - yield {"survey_id": survey["id"]} + return f"surveys/{stream_slice['survey_id']}/responses/bulk" def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: """ @@ -244,9 +222,21 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late and returning an updated state object. """ survey_id = latest_record.get("survey_id") - return { - self.cursor_field: max(latest_record.get(self.cursor_field), current_stream_state.get(survey_id, {}).get(self.cursor_field, "")) - } + if not current_stream_state: + current_stream_state = {} + survey_state = current_stream_state.get(survey_id, {}) + + latest_record_value = latest_record.get(self.cursor_field, "") + if latest_record_value: + # add 1 second, otherwise next incremental syns return the same record + latest_record_value = pendulum.parse(latest_record_value).add(seconds=1).to_iso8601_string() + + state_value = max( + latest_record_value, + survey_state.get(self.cursor_field, ""), + ) + current_stream_state[survey_id] = {self.cursor_field: state_value} + return current_stream_state def request_params(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: params = super().request_params(stream_state=stream_state, **kwargs) diff --git a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_source.py b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_source.py new file mode 100644 index 000000000000..cd5c30460fc0 --- /dev/null +++ b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_source.py @@ -0,0 +1,28 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from source_surveymonkey.source import SourceSurveymonkey + +source_config = {"start_date": "2021-01-01T00:00:00", "access_token": "something"} + + +def test_source_streams(): + streams = SourceSurveymonkey().streams(config=source_config) + assert len(streams) == 4 + + +def test_source_check_connection(requests_mock): + requests_mock.get( + "https://api.surveymonkey.com/v3/users/me", json={"scopes": {"granted": ["responses_read_detail", "surveys_read", "users_read"]}} + ) + + results = SourceSurveymonkey().check_connection(logger=None, config=source_config) + assert results == (True, None) + + +def test_source_check_connection_failed(requests_mock): + requests_mock.get("https://api.surveymonkey.com/v3/users/me", json={"scopes": {"granted": ["surveys_read", "users_read"]}}) + + results = SourceSurveymonkey().check_connection(logger=None, config=source_config) + assert results == (False, "missed required scopes: responses_read_detail") diff --git a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py new file mode 100644 index 000000000000..ee5ac1e57fb4 --- /dev/null +++ b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py @@ -0,0 +1,414 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import Mock + +import pendulum +import pytest +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.http.auth import NoAuth +from source_surveymonkey.streams import SurveyIds, SurveyPages, SurveyQuestions, SurveyResponses, Surveys + +args_mock = {"authenticator": NoAuth(), "start_date": pendulum.parse("2000-01-01"), "survey_ids": []} + +records_survey_ids = [ + { + "id": "307785415", + "title": "b9jo5h23l7pa", + "nickname": "qhs5vg2qi0o4arsjiwy2ay00n82n", + "href": "https://api.surveymonkey.com/v3/surveys/307785415", + }, + { + "id": "307785388", + "title": "igpfp2yfsw90df6nxbsb49v", + "nickname": "h23gl22ulmfsyt4q7xt", + "href": "https://api.surveymonkey.com/v3/surveys/307785388", + }, +] + +response_survey_ids = { + "data": records_survey_ids, + "per_page": 50, + "page": 1, + "total": 2, + "links": {"self": "https://api.surveymonkey.com/v3/surveys?per_page=50&page=1"}, +} + + +def test_survey_ids(requests_mock): + requests_mock.get("https://api.surveymonkey.com/v3/surveys", json=response_survey_ids) + stream = SurveyIds(**args_mock) + records = stream.read_records(sync_mode=SyncMode.full_refresh) + assert list(records) == records_survey_ids + + +def test_user_defined_retry(requests_mock): + requests_mock.get( + "https://api.surveymonkey.com/v3/surveys", + [ + { + "status_code": 429, + "headers": {"X-Ratelimit-App-Global-Minute-Remaining": "0"}, + "json": { + "error": { + "id": 1040, + "name": "Rate limit reached", + "docs": "https://developer.surveymonkey.com/api/v3/#error-codes", + "message": "Too many requests were made, try again later.", + "http_status_code": 429, + } + }, + }, + {"status_code": 200, "headers": {"X-Ratelimit-App-Global-Minute-Remaining": "100"}, "json": response_survey_ids}, + ], + ) + + stream = SurveyIds(**args_mock) + stream.default_backoff_time = 3 + records = stream.read_records(sync_mode=SyncMode.full_refresh) + assert list(records) == records_survey_ids + + +# def test_cache_usage(requests_mock): +# cache_file_name = source_surveymonkey.streams.cache_file.name +# requests_mock.get("https://api.surveymonkey.com/v3/surveys", [ +# {'status_code': 200, +# 'headers': {'X-Ratelimit-App-Global-Minute-Remaining': '100'}, +# 'json': response_survey_ids} +# ]) +# +# stream = SurveyIds(**args_mock) +# records = stream.read_records(sync_mode=SyncMode.full_refresh) +# assert list(records) == records_survey_ids +# +# records = stream.read_records(sync_mode=SyncMode.full_refresh) +# assert list(records) == records_survey_ids +# +# assert requests_mock.call_count == 2 +# +# +# with vcr.use_cassette(cache_file_name, record_mode=None, serializer="json", decode_compressed_response=True) as cass: +# # assert cass.requests == 0 +# # assert cass.responses == 0 +# assert cass.responses_of('https://api.surveymonkey.com/v3/surveys') == 0 + + +def test_slices_from_survey_ids(requests_mock): + requests_mock.get("https://api.surveymonkey.com/v3/surveys", json=response_survey_ids) + stream_slices = Surveys(**args_mock).stream_slices() + assert list(stream_slices) == [{"survey_id": "307785415"}, {"survey_id": "307785388"}] + + +def test_slices_from_config(requests_mock): + args = {**args_mock, **{"survey_ids": ["307785415"]}} + stream_slices = Surveys(**args).stream_slices() + assert list(stream_slices) == [{"survey_id": "307785415"}] + + +response_survey_details = { + "title": "b9jo5h23l7pa", + "nickname": "qhs5vg2qi0o4arsjiwy2ay00n82n", + "language": "ru", + "folder_id": "0", + "category": "", + "question_count": 10, + "page_count": 3, + "response_count": 20, + "date_created": "2021-06-09T21:20:00", + "date_modified": "2021-06-10T11:07:00", + "id": "307785415", + "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, + "is_owner": True, + "footer": True, + "theme_id": "4510354", + "custom_variables": {}, + "href": "https://api.surveymonkey.com/v3/surveys/307785415", + "analyze_url": "https://www.surveymonkey.com/analyze/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D", + "edit_url": "https://www.surveymonkey.com/create/?sm=BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D", + "collect_url": "https://www.surveymonkey.com/collect/list?sm=BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D", + "summary_url": "https://www.surveymonkey.com/summary/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D", + "preview": "https://www.surveymonkey.com/r/Preview/?sm=YVdtL_2BP5oiGTrfksyofvENkBr7v87Xfh8hbcJr8rbqgesWvwJjz5N1F7pCSRcDoy", + "pages": [ + { + "title": "", + "description": "", + "position": 1, + "question_count": 0, + "id": "168831392", + "href": "https://api.surveymonkey.com/v3/surveys/307785415/pages/168831392", + "questions": [], + }, + { + "title": "p71uerk2uh7k5", + "description": "92cb9d98j15jmfo", + "position": 2, + "question_count": 2, + "id": "168831393", + "href": "https://api.surveymonkey.com/v3/surveys/307785415/pages/168831393", + "questions": [ + { + "id": "667461690", + "position": 1, + "visible": True, + "family": "single_choice", + "subtype": "vertical", + "layout": None, + "sorting": None, + "required": None, + "validation": None, + "forced_ranking": False, + "headings": [{"heading": "53o3ibly at73qjs4e4 y9dug7jxfmpmr 8esacb5"}], + "href": "https://api.surveymonkey.com/v3/surveys/307785415/pages/168831393/questions/667461690", + "answers": { + "choices": [ + { + "position": 1, + "visible": True, + "text": "lg2mcft4e64 ywiatkmeo ci3rr4l2v0 ot6un49a 4b28sq4g8qv7tj 4ihpko73bp0k6lf swaeo3o4mg2jf5g rnh225wj520w1ps p9emk1wg64vwl", + "quiz_options": {"score": 0}, + "id": "4385174700", + }, + { + "position": 2, + "visible": True, + "text": "ywg8bovna adsahna5kd1jg vdism1 w045ovutkx9 oubne2u vd0x7lh3 y3npa4kfb5", + "quiz_options": {"score": 0}, + "id": "4385174701", + }, + ] + }, + }, + { + "id": "667461777", + "position": 2, + "visible": True, + "family": "single_choice", + "subtype": "menu", + "layout": None, + "sorting": None, + "required": None, + "validation": None, + "forced_ranking": False, + "headings": [{"heading": "kjqdk eo7hfnu or7bmd1iwqxxp sguqta4f8141iy"}], + "href": "https://api.surveymonkey.com/v3/surveys/307785415/pages/168831393/questions/667461777", + "answers": { + "choices": [ + { + "position": 1, + "visible": True, + "text": "11bp1ll11nu0 ool67 tkbke01j3mtq 22f4r54u073p h6kt4puolum4", + "quiz_options": {"score": 0}, + "id": "4385174970", + }, + { + "position": 2, + "visible": True, + "text": "8q53omsxw8 08yyjvj3ns9j yu7yap87 d2tgjv55j5d5o3y dbd69m94qav1wma 8upqf7cliu hb26pytfkwyt rfo2ac4", + "quiz_options": {"score": 0}, + "id": "4385174971", + }, + ] + }, + }, + ], + }, + ], +} + + +def test_surveys(requests_mock): + requests_mock.get("https://api.surveymonkey.com/v3/surveys/307785415/details", json=response_survey_details) + args = {**args_mock, **{"survey_ids": ["307785415"]}} + records = Surveys(**args).read_records(sync_mode=SyncMode.full_refresh, stream_slice={"survey_id": "307785415"}) + assert list(records) == [ + { + "analyze_url": "https://www.surveymonkey.com/analyze/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D", + "buttons_text": {"done_button": "Nax_Don_Gon!", "exit_button": "", "next_button": "Nex >>>>>", "prev_button": "Nix <<<<<"}, + "category": "", + "collect_url": "https://www.surveymonkey.com/collect/list?sm=BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D", + "custom_variables": {}, + "date_created": "2021-06-09T21:20:00", + "date_modified": "2021-06-10T11:07:00", + "edit_url": "https://www.surveymonkey.com/create/?sm=BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D", + "folder_id": "0", + "footer": True, + "href": "https://api.surveymonkey.com/v3/surveys/307785415", + "id": "307785415", + "is_owner": True, + "language": "ru", + "nickname": "qhs5vg2qi0o4arsjiwy2ay00n82n", + "page_count": 3, + "preview": "https://www.surveymonkey.com/r/Preview/?sm=YVdtL_2BP5oiGTrfksyofvENkBr7v87Xfh8hbcJr8rbqgesWvwJjz5N1F7pCSRcDoy", + "question_count": 10, + "response_count": 20, + "summary_url": "https://www.surveymonkey.com/summary/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D", + "theme_id": "4510354", + "title": "b9jo5h23l7pa", + } + ] + + +def test_survey_pages(requests_mock): + requests_mock.get("https://api.surveymonkey.com/v3/surveys/307785415/details", json=response_survey_details) + args = {**args_mock, **{"survey_ids": ["307785415"]}} + records = SurveyPages(**args).read_records(sync_mode=SyncMode.full_refresh, stream_slice={"survey_id": "307785415"}) + assert list(records) == [ + { + "description": "", + "href": "https://api.surveymonkey.com/v3/surveys/307785415/pages/168831392", + "id": "168831392", + "position": 1, + "question_count": 0, + "title": "", + }, + { + "description": "92cb9d98j15jmfo", + "href": "https://api.surveymonkey.com/v3/surveys/307785415/pages/168831393", + "id": "168831393", + "position": 2, + "question_count": 2, + "title": "p71uerk2uh7k5", + }, + ] + + +def test_survey_questions(requests_mock): + requests_mock.get("https://api.surveymonkey.com/v3/surveys/307785415/details", json=response_survey_details) + args = {**args_mock, **{"survey_ids": ["307785415"]}} + records = SurveyQuestions(**args).read_records(sync_mode=SyncMode.full_refresh, stream_slice={"survey_id": "307785415"}) + assert list(records) == [ + { + "answers": { + "choices": [ + { + "id": "4385174700", + "position": 1, + "quiz_options": {"score": 0}, + "text": "lg2mcft4e64 ywiatkmeo ci3rr4l2v0 ot6un49a " + "4b28sq4g8qv7tj 4ihpko73bp0k6lf " + "swaeo3o4mg2jf5g rnh225wj520w1ps " + "p9emk1wg64vwl", + "visible": True, + }, + { + "id": "4385174701", + "position": 2, + "quiz_options": {"score": 0}, + "text": "ywg8bovna adsahna5kd1jg vdism1 w045ovutkx9 " "oubne2u vd0x7lh3 y3npa4kfb5", + "visible": True, + }, + ] + }, + "family": "single_choice", + "forced_ranking": False, + "headings": [{"heading": "53o3ibly at73qjs4e4 y9dug7jxfmpmr 8esacb5"}], + "href": "https://api.surveymonkey.com/v3/surveys/307785415/pages/168831393/questions/667461690", + "id": "667461690", + "layout": None, + "page_id": "168831393", + "position": 1, + "required": None, + "sorting": None, + "subtype": "vertical", + "validation": None, + "visible": True, + }, + { + "answers": { + "choices": [ + { + "id": "4385174970", + "position": 1, + "quiz_options": {"score": 0}, + "text": "11bp1ll11nu0 ool67 tkbke01j3mtq " "22f4r54u073p h6kt4puolum4", + "visible": True, + }, + { + "id": "4385174971", + "position": 2, + "quiz_options": {"score": 0}, + "text": "8q53omsxw8 08yyjvj3ns9j yu7yap87 " "d2tgjv55j5d5o3y dbd69m94qav1wma 8upqf7cliu " "hb26pytfkwyt rfo2ac4", + "visible": True, + }, + ] + }, + "family": "single_choice", + "forced_ranking": False, + "headings": [{"heading": "kjqdk eo7hfnu or7bmd1iwqxxp sguqta4f8141iy"}], + "href": "https://api.surveymonkey.com/v3/surveys/307785415/pages/168831393/questions/667461777", + "id": "667461777", + "layout": None, + "page_id": "168831393", + "position": 2, + "required": None, + "sorting": None, + "subtype": "menu", + "validation": None, + "visible": True, + }, + ] + + +def test_surveys_next_page_token(): + args = {**args_mock, **{"survey_ids": ["307785415"]}} + stream = SurveyIds(**args) + + mockresponse = Mock() + mockresponse.json.return_value = { + "links": { + "self": "https://api.surveymonkey.com/v3/surveys?page=1&per_page=50", + "next": "https://api.surveymonkey.com/v3/surveys?page=2&per_page=50", + "last": "https://api.surveymonkey.com/v3/surveys?page=5&per_page=50", + } + } + + params = stream.next_page_token(mockresponse) + assert params == {"page": "2", "per_page": "50"} + + +@pytest.mark.parametrize( + "current_stream_state,latest_record,state", + [ + ( + {"307785415": {"date_modified": "2021-01-01T00:00:00+00:00"}}, + {"survey_id": "307785415", "date_modified": "2021-12-01T00:00:00+00:00"}, + {"307785415": {"date_modified": "2021-12-01T00:00:01+00:00"}}, + ), + ( + {}, + {"survey_id": "307785415", "date_modified": "2021-12-01T00:00:00+00:00"}, + {"307785415": {"date_modified": "2021-12-01T00:00:01+00:00"}}, + ), + ( + {"307785415": {"date_modified": "2021-01-01T00:00:00+00:00"}}, + {"survey_id": "307785415"}, + {"307785415": {"date_modified": "2021-01-01T00:00:00+00:00"}}, + ), + ], +) +def test_surveys_responses_get_updated_state(current_stream_state, latest_record, state): + args = {**args_mock, **{"survey_ids": ["307785415"]}} + stream = SurveyResponses(**args) + actual_state = stream.get_updated_state(current_stream_state=current_stream_state, latest_record=latest_record) + assert actual_state == state + + +@pytest.mark.parametrize( + "stream_state,params", + [ + ( + {"307785415": {"date_modified": "2021-01-01T00:00:00+00:00"}}, + {"start_modified_at": "2021-01-01T00:00:00"}, + ), + ( + {}, + {"start_modified_at": "2000-01-01T00:00:00"}, # return start_date + ), + ], +) +def test_surveys_responses_request_params(stream_state, params): + args = {**args_mock, **{"survey_ids": ["307785415"]}} + stream = SurveyResponses(**args) + actual_params = stream.request_params(stream_state=stream_state, stream_slice={"survey_id": "307785415"}) + assert actual_params == params From cf38fb7bb68fb792d286a316fd22d8f2aac3a85b Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Thu, 28 Jul 2022 21:52:00 +0300 Subject: [PATCH 2/8] removed comments --- .../unit_tests/test_streams.py | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py index ee5ac1e57fb4..30d3df67b8a8 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py @@ -70,30 +70,6 @@ def test_user_defined_retry(requests_mock): assert list(records) == records_survey_ids -# def test_cache_usage(requests_mock): -# cache_file_name = source_surveymonkey.streams.cache_file.name -# requests_mock.get("https://api.surveymonkey.com/v3/surveys", [ -# {'status_code': 200, -# 'headers': {'X-Ratelimit-App-Global-Minute-Remaining': '100'}, -# 'json': response_survey_ids} -# ]) -# -# stream = SurveyIds(**args_mock) -# records = stream.read_records(sync_mode=SyncMode.full_refresh) -# assert list(records) == records_survey_ids -# -# records = stream.read_records(sync_mode=SyncMode.full_refresh) -# assert list(records) == records_survey_ids -# -# assert requests_mock.call_count == 2 -# -# -# with vcr.use_cassette(cache_file_name, record_mode=None, serializer="json", decode_compressed_response=True) as cass: -# # assert cass.requests == 0 -# # assert cass.responses == 0 -# assert cass.responses_of('https://api.surveymonkey.com/v3/surveys') == 0 - - def test_slices_from_survey_ids(requests_mock): requests_mock.get("https://api.surveymonkey.com/v3/surveys", json=response_survey_ids) stream_slices = Surveys(**args_mock).stream_slices() From 87b9f1d1b6f154263cd4571d944af0a4ec502be3 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Thu, 28 Jul 2022 23:03:49 +0300 Subject: [PATCH 3/8] updated docs --- docs/integrations/sources/surveymonkey.md | 86 +++++++++++------------ 1 file changed, 42 insertions(+), 44 deletions(-) diff --git a/docs/integrations/sources/surveymonkey.md b/docs/integrations/sources/surveymonkey.md index f523c8049fda..f10e122ae588 100644 --- a/docs/integrations/sources/surveymonkey.md +++ b/docs/integrations/sources/surveymonkey.md @@ -1,34 +1,43 @@ # SurveyMonkey -## Sync overview +This page guides you through the process of setting up the SurveyMonkey source connector. -This source can sync data for the [SurveyMonkey API](https://developer.surveymonkey.com/api/v3/). It supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. +## Prerequisites -### Output schema + ### For Airbyte OSS: +* Access Token -This Source is capable of syncing the following core Streams: +## Setup guide +### Step 1: Set up SurveyMonkey +Please read this [docs](https://developer.surveymonkey.com/api/v3/#getting-started). Register your application [here](https://developer.surveymonkey.com/apps/) Then go to Settings and copy your access token -* [Surveys](https://developer.surveymonkey.com/api/v3/#surveys) \(Incremental\) -* [SurveyPages](https://developer.surveymonkey.com/api/v3/#surveys-id-pages) -* [SurveyQuestions](https://developer.surveymonkey.com/api/v3/#surveys-id-pages-id-questions) -* [SurveyResponses](https://developer.surveymonkey.com/api/v3/#survey-responses) +## Step 2: Set up the source connector in Airbyte -### Data type mapping +**For Airbyte Cloud:** -| Integration Type | Airbyte Type | Notes | -| :--- | :--- | :--- | -| `string` | `string` | | -| `number` | `number` | | -| `array` | `array` | | -| `object` | `object` | | +1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. +3. On the source setup page, select **SurveyMonkey** from the Source type dropdown and enter a name for this connector. +4. lick `Authenticate your account`. +5. Log in and Authorize to the SurveyMonkey account +6. Choose required Start date +7. click `Set up source`. -### Features +**For Airbyte OSS:** -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental Sync | Yes | | -| Namespaces | No | | +1. Go to local Airbyte page. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. +3. On the source setup page, select **SurveyMonkey** from the Source type dropdown and enter a name for this connector. +4. Add **Access Token** +5. Choose required Start date +6. Click `Set up source`. + +## Supported streams and sync modes + +* [Surveys](https://developer.surveymonkey.com/api/v3/#surveys) \(Incremental\) +* [SurveyPages](https://developer.surveymonkey.com/api/v3/#surveys-id-pages) +* [SurveyQuestions](https://developer.surveymonkey.com/api/v3/#surveys-id-pages-id-questions) +* [SurveyResponses](https://developer.surveymonkey.com/api/v3/#survey-responses) \(Incremental\) ### Performance considerations @@ -39,29 +48,18 @@ The SurveyMonkey API applies heavy API quotas for default private apps, which ha To cover more data from this source we use caching. -Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. - -## Getting started - -### Requirements - -* SurveyMonkey API Key - -### Setup guide - -Please read this [docs](https://developer.surveymonkey.com/api/v3/#getting-started). Register your application [here](https://developer.surveymonkey.com/apps/) Then go to Settings and copy your access token - ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:-------------------------------------------------------|:--------------------------------------------------| -| 0.1.8 | 2022-05-20 | [13046](https://github.com/airbytehq/airbyte/pull/13046) | Fix incremental streams | -| 0.1.7 | 2022-02-24 | [8768](https://github.com/airbytehq/airbyte/pull/8768) | Add custom survey IDs to limit API calls | -| 0.1.6 | 2022-01-14 | [9508](https://github.com/airbytehq/airbyte/pull/9508) | Scopes change | -| 0.1.5 | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628) | Update fields in source-connectors specifications | -| 0.1.4 | 2021-11-11 | [7868](https://github.com/airbytehq/airbyte/pull/7868) | Improve 'check' using '/users/me' API call | -| 0.1.3 | 2021-11-01 | [7433](https://github.com/airbytehq/airbyte/pull/7433) | Remove unsused oAuth flow parameters | -| 0.1.2 | 2021-10-27 | [7433](https://github.com/airbytehq/airbyte/pull/7433) | Add OAuth support | -| 0.1.1 | 2021-09-10 | [5983](https://github.com/airbytehq/airbyte/pull/5983) | Fix caching for gzip compressed http response | -| 0.1.0 | 2021-07-06 | [4097](https://github.com/airbytehq/airbyte/pull/4097) | Initial Release | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------| +| 0.1.9 | 2022-07-28 | [13046](https://github.com/airbytehq/airbyte/pull/14998) | fixed state for response stream, fixed backoff behaviour, added unittest | +| 0.1.8 | 2022-05-20 | [13046](https://github.com/airbytehq/airbyte/pull/13046) | Fix incremental streams | +| 0.1.7 | 2022-02-24 | [8768](https://github.com/airbytehq/airbyte/pull/8768) | Add custom survey IDs to limit API calls | +| 0.1.6 | 2022-01-14 | [9508](https://github.com/airbytehq/airbyte/pull/9508) | Scopes change | +| 0.1.5 | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628) | Update fields in source-connectors specifications | +| 0.1.4 | 2021-11-11 | [7868](https://github.com/airbytehq/airbyte/pull/7868) | Improve 'check' using '/users/me' API call | +| 0.1.3 | 2021-11-01 | [7433](https://github.com/airbytehq/airbyte/pull/7433) | Remove unsused oAuth flow parameters | +| 0.1.2 | 2021-10-27 | [7433](https://github.com/airbytehq/airbyte/pull/7433) | Add OAuth support | +| 0.1.1 | 2021-09-10 | [5983](https://github.com/airbytehq/airbyte/pull/5983) | Fix caching for gzip compressed http response | +| 0.1.0 | 2021-07-06 | [4097](https://github.com/airbytehq/airbyte/pull/4097) | Initial Release | From 70510536e82c830b63ca61aac6e0f704b34e5afc Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 2 Aug 2022 18:24:37 +0300 Subject: [PATCH 4/8] updated docs --- .../source-surveymonkey/source_surveymonkey/streams.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py index 7df340300d21..69ffed452c85 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py +++ b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py @@ -135,7 +135,7 @@ def request_params(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMa return params -class SliceMixin: +class SurveyIDSliceMixin: def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: return f"surveys/{stream_slice['survey_id']}/details" @@ -148,7 +148,7 @@ def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs): yield {"survey_id": survey["id"]} -class Surveys(SliceMixin, IncrementalSurveymonkeyStream): +class Surveys(SurveyIDSliceMixin, IncrementalSurveymonkeyStream): """ Docs: https://developer.surveymonkey.com/api/v3/#surveys A source for stream slices. It does not contain useful info itself. @@ -179,7 +179,7 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, yield record -class SurveyPages(SliceMixin, SurveymonkeyStream): +class SurveyPages(SurveyIDSliceMixin, SurveymonkeyStream): """should be filled from SurveyDetails""" data_field = "pages" @@ -191,7 +191,7 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, yield record -class SurveyQuestions(SliceMixin, SurveymonkeyStream): +class SurveyQuestions(SurveyIDSliceMixin, SurveymonkeyStream): """should be filled from SurveyDetails""" data_field = "pages" @@ -206,7 +206,7 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, yield question -class SurveyResponses(SliceMixin, IncrementalSurveymonkeyStream): +class SurveyResponses(SurveyIDSliceMixin, IncrementalSurveymonkeyStream): """ Docs: https://developer.surveymonkey.com/api/v3/#survey-responses """ From 73c85dc1ef18cd2c95cd37256a47fb63ffdee209 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Wed, 3 Aug 2022 19:16:53 +0300 Subject: [PATCH 5/8] bumped connector version --- airbyte-integrations/connectors/source-surveymonkey/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-surveymonkey/Dockerfile b/airbyte-integrations/connectors/source-surveymonkey/Dockerfile index a1a7edc2ef47..7197f07dc25e 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/Dockerfile +++ b/airbyte-integrations/connectors/source-surveymonkey/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.8 +LABEL io.airbyte.version=0.1.9 LABEL io.airbyte.name=airbyte/source-surveymonkey From a1049fc6575838c21ddaa7134cd9a52fc8751835 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Wed, 3 Aug 2022 19:48:15 +0300 Subject: [PATCH 6/8] bumped release stage --- .../init/src/main/resources/seed/source_definitions.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 0a8e8bdac57a..f47f338360e0 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -945,11 +945,11 @@ - name: SurveyMonkey sourceDefinitionId: badc5925-0485-42be-8caa-b34096cb71b5 dockerRepository: airbyte/source-surveymonkey - dockerImageTag: 0.1.8 + dockerImageTag: 0.1.9 documentationUrl: https://docs.airbyte.io/integrations/sources/surveymonkey icon: surveymonkey.svg sourceType: api - releaseStage: alpha + releaseStage: beta - name: TalkDesk Explore sourceDefinitionId: f00d2cf4-3c28-499a-ba93-b50b6f26359e dockerRepository: airbyte/source-talkdesk-explore From 0e8e73be3353a774b843a618fee56fe5fb271df1 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Wed, 3 Aug 2022 17:40:35 +0000 Subject: [PATCH 7/8] auto-bump connector version [ci skip] --- .../src/main/resources/seed/source_specs.yaml | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 5ac83f89b70c..48ba71a473b5 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -9338,7 +9338,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-surveymonkey:0.1.8" +- dockerImage: "airbyte/source-surveymonkey:0.1.9" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/surveymonkey" connectionSpecification: @@ -9346,26 +9346,30 @@ title: "SurveyMonkey Spec" type: "object" required: + - "access_token" - "start_date" additionalProperties: true properties: + access_token: + title: "Access Token" + order: 0 + type: "string" + airbyte_secret: true + description: "Access Token for making authenticated requests. See the docs\ + \ for information on how to generate this key." start_date: title: "Start Date" + order: 1 type: "string" description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\ \ data before this date will not be replicated." pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z?$" examples: - "2021-01-01T00:00:00Z" - access_token: - title: "Access Token" - type: "string" - airbyte_secret: true - description: "Access Token for making authenticated requests. See the docs\ - \ for information on how to generate this key." survey_ids: type: "array" + order: 2 items: type: "string" pattern: "^[0-9]{8,9}$" From 19768a673b922af72f046a37ca858fe4a6f4c637 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Wed, 3 Aug 2022 20:43:41 +0300 Subject: [PATCH 8/8] updated source_specs.yaml --- .../src/main/resources/seed/source_specs.yaml | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 5ac83f89b70c..48ba71a473b5 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -9338,7 +9338,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-surveymonkey:0.1.8" +- dockerImage: "airbyte/source-surveymonkey:0.1.9" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/surveymonkey" connectionSpecification: @@ -9346,26 +9346,30 @@ title: "SurveyMonkey Spec" type: "object" required: + - "access_token" - "start_date" additionalProperties: true properties: + access_token: + title: "Access Token" + order: 0 + type: "string" + airbyte_secret: true + description: "Access Token for making authenticated requests. See the docs\ + \ for information on how to generate this key." start_date: title: "Start Date" + order: 1 type: "string" description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\ \ data before this date will not be replicated." pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z?$" examples: - "2021-01-01T00:00:00Z" - access_token: - title: "Access Token" - type: "string" - airbyte_secret: true - description: "Access Token for making authenticated requests. See the docs\ - \ for information on how to generate this key." survey_ids: type: "array" + order: 2 items: type: "string" pattern: "^[0-9]{8,9}$"