From cf90197b7a2a6c190ecb22082c73cb7b05a11d25 Mon Sep 17 00:00:00 2001 From: Sherif Nada Date: Mon, 19 Jul 2021 14:28:33 -0700 Subject: [PATCH 1/6] add debug log statements for sendgrid --- .../source_sendgrid/streams.py | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py index c8c2a7de1b16..24a8c5902868 100644 --- a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py +++ b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py @@ -48,8 +48,22 @@ def parse_response( ) -> Iterable[Mapping]: json_response = response.json() records = json_response.get(self.data_field, []) if self.data_field is not None else json_response - for record in records: - yield record + + if records is None: + for record in records: + yield record + # else: + # TODO sendgrid's API is sending empty (not empty array, just empty) responses at times. It's not entirely clear why, but adding these + # log statements to help reproduce and prevent the connector from failing + err_msg = f"Response contained no valid JSON data. Response body: {response.text}\n"\ + f"Response status: {response.status_code}\n"\ + f"Response body: {response.text}\n"\ + f"Response headers: {response.headers}\n"\ + f"Request URL: {response.request.url}\n"\ + f"Request body: {response.request.body}\n" + # do NOT print request headers as it contains auth token + self.logger.info(err_msg) + class SendgridStreamOffsetPagination(SendgridStream): @@ -72,7 +86,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, return {"offset": self.offset} -class SendgridStreamIncrementalMixin(HttpStream): +class SendgridStreamIncrementalMixin(HttpStream, ABC): cursor_field = "created" def __init__(self, start_time: int, **kwargs): From 59d16c19bd88c7e9c507d8d9098e30db4a08f019 Mon Sep 17 00:00:00 2001 From: Sherif Nada Date: Mon, 19 Jul 2021 14:49:48 -0700 Subject: [PATCH 2/6] add spoofed integration test --- .../connectors/source-sendgrid/Dockerfile | 2 +- .../integration_tests/integration_test.py | 26 +++++++++++++++++++ .../source_sendgrid/streams.py | 24 ++++++++--------- 3 files changed, 39 insertions(+), 13 deletions(-) create mode 100644 airbyte-integrations/connectors/source-sendgrid/integration_tests/integration_test.py diff --git a/airbyte-integrations/connectors/source-sendgrid/Dockerfile b/airbyte-integrations/connectors/source-sendgrid/Dockerfile index 6e7a58ae1537..1daa4c4657f6 100644 --- a/airbyte-integrations/connectors/source-sendgrid/Dockerfile +++ b/airbyte-integrations/connectors/source-sendgrid/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.5 +LABEL io.airbyte.version=0.2.6-rc1 LABEL io.airbyte.name=airbyte/source-sendgrid diff --git a/airbyte-integrations/connectors/source-sendgrid/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-sendgrid/integration_tests/integration_test.py new file mode 100644 index 000000000000..4349a4880fb9 --- /dev/null +++ b/airbyte-integrations/connectors/source-sendgrid/integration_tests/integration_test.py @@ -0,0 +1,26 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +def test_example(): + assert True diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py index 24a8c5902868..2bc36ef310f4 100644 --- a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py +++ b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py @@ -49,20 +49,20 @@ def parse_response( json_response = response.json() records = json_response.get(self.data_field, []) if self.data_field is not None else json_response - if records is None: + if records is not None: for record in records: yield record - # else: - # TODO sendgrid's API is sending empty (not empty array, just empty) responses at times. It's not entirely clear why, but adding these - # log statements to help reproduce and prevent the connector from failing - err_msg = f"Response contained no valid JSON data. Response body: {response.text}\n"\ - f"Response status: {response.status_code}\n"\ - f"Response body: {response.text}\n"\ - f"Response headers: {response.headers}\n"\ - f"Request URL: {response.request.url}\n"\ - f"Request body: {response.request.body}\n" - # do NOT print request headers as it contains auth token - self.logger.info(err_msg) + else: + # TODO sendgrid's API is sending empty (not empty array, just empty) responses at times. It's not entirely clear why, but adding these + # log statements to help reproduce and prevent the connector from failing + err_msg = f"Response contained no valid JSON data. Response body: {response.text}\n"\ + f"Response status: {response.status_code}\n"\ + f"Response body: {response.text}\n"\ + f"Response headers: {response.headers}\n"\ + f"Request URL: {response.request.url}\n"\ + f"Request body: {response.request.body}\n" + # do NOT print request headers as it contains auth token + self.logger.info(err_msg) From 959deb5d505513df47c19b68570f1bed864c665a Mon Sep 17 00:00:00 2001 From: "Sherif A. Nada" Date: Mon, 19 Jul 2021 18:15:08 -0700 Subject: [PATCH 3/6] Update streams.py --- .../connectors/source-sendgrid/source_sendgrid/streams.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py index 2bc36ef310f4..ce2a7b6b84cf 100644 --- a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py +++ b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py @@ -53,8 +53,8 @@ def parse_response( for record in records: yield record else: - # TODO sendgrid's API is sending empty (not empty array, just empty) responses at times. It's not entirely clear why, but adding these - # log statements to help reproduce and prevent the connector from failing + # TODO sendgrid's API is sending null responses at times. This seems like a bug on the API side, so we're adding + # log statements to help reproduce and prevent the connector from failing. err_msg = f"Response contained no valid JSON data. Response body: {response.text}\n"\ f"Response status: {response.status_code}\n"\ f"Response body: {response.text}\n"\ From 14266803eac7fd82794ddad071074cb247e84800 Mon Sep 17 00:00:00 2001 From: Sherif Nada Date: Mon, 19 Jul 2021 18:19:06 -0700 Subject: [PATCH 4/6] update docs, version --- .../fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87.json | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-integrations/connectors/source-sendgrid/Dockerfile | 2 +- airbyte-integrations/connectors/source-sendgrid/README.md | 3 +++ docs/integrations/sources/sendgrid.md | 3 +++ 5 files changed, 9 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87.json index 19492bfa75fd..40b430c05cf8 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87", "name": "Sendgrid", "dockerRepository": "airbyte/source-sendgrid", - "dockerImageTag": "0.2.5", + "dockerImageTag": "0.2.6", "documentationUrl": "https://hub.docker.com/r/airbyte/source-sendgrid", "icon": "sendgrid.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 7c2fa0bdfd33..a65744f97bf5 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -73,7 +73,7 @@ - sourceDefinitionId: fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87 name: Sendgrid dockerRepository: airbyte/source-sendgrid - dockerImageTag: 0.2.5 + dockerImageTag: 0.2.6 documentationUrl: https://hub.docker.com/r/airbyte/source-sendgrid icon: sendgrid.svg - sourceDefinitionId: 9e0556f4-69df-4522-a3fb-03264d36b348 diff --git a/airbyte-integrations/connectors/source-sendgrid/Dockerfile b/airbyte-integrations/connectors/source-sendgrid/Dockerfile index 1daa4c4657f6..944dfc4ad464 100644 --- a/airbyte-integrations/connectors/source-sendgrid/Dockerfile +++ b/airbyte-integrations/connectors/source-sendgrid/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.6-rc1 +LABEL io.airbyte.version=0.2.6 LABEL io.airbyte.name=airbyte/source-sendgrid diff --git a/airbyte-integrations/connectors/source-sendgrid/README.md b/airbyte-integrations/connectors/source-sendgrid/README.md index e413ad32ce73..bb171e8b10cd 100644 --- a/airbyte-integrations/connectors/source-sendgrid/README.md +++ b/airbyte-integrations/connectors/source-sendgrid/README.md @@ -98,3 +98,6 @@ You've checked out the repo, implemented a million dollar feature, and you're re 1. Create a Pull Request 1. Pat yourself on the back for being an awesome contributor 1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master + +### Changelog +See the [docs](https://docs.airbyte.io/integrations/sources/sendgrid#changelog) for the changelog. diff --git a/docs/integrations/sources/sendgrid.md b/docs/integrations/sources/sendgrid.md index 1de5acf45850..7f83e4870cdf 100644 --- a/docs/integrations/sources/sendgrid.md +++ b/docs/integrations/sources/sendgrid.md @@ -41,3 +41,6 @@ Generate a API key using the [Sendgrid documentation](https://sendgrid.com/docs/ We recommend creating a key specifically for Airbyte access. This will allow you to control which resources Airbyte should be able to access. The API key should be read-only on all resources except Marketing, where it needs Full Access. +| Version | Date | Pull Request | Subject | +| :------ | :-------- | :----- | :------ | +| 0.2.6 | 2021-07-19 | [4839](https://github.com/airbytehq/airbyte/pull/4839) | Gracefully handle malformed responses from the API | From bb390cc2e4af25a4432fb338be1dcf337dc70d36 Mon Sep 17 00:00:00 2001 From: Sherif Nada Date: Mon, 19 Jul 2021 18:33:18 -0700 Subject: [PATCH 5/6] unit_tests --- .../connectors/source-sendgrid/setup.py | 2 +- .../source-sendgrid/unit_tests/unit_test.py | 20 ++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-sendgrid/setup.py b/airbyte-integrations/connectors/source-sendgrid/setup.py index 184ec5e79757..6de1434caed9 100644 --- a/airbyte-integrations/connectors/source-sendgrid/setup.py +++ b/airbyte-integrations/connectors/source-sendgrid/setup.py @@ -31,6 +31,6 @@ author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-cdk~=0.1", "backoff", "requests", "pytest==6.1.2"], + install_requires=["airbyte-cdk~=0.1", "backoff", "requests", "pytest==6.1.2", "pytest-mock"], package_data={"": ["*.json", "schemas/*.json"]}, ) diff --git a/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py index 53942b8a6d2b..9715b4661054 100644 --- a/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py @@ -21,10 +21,28 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # +from unittest.mock import MagicMock - +import pytest +import requests from airbyte_cdk.logger import AirbyteLogger from source_sendgrid.source import SourceSendgrid +from source_sendgrid.streams import SendgridStream + + +@pytest.fixture(name="sendgrid_stream") +def sendgrid_stream_fixture(mocker) -> SendgridStream: + # Wipe the internal list of abstract methods to allow instantiating the abstract class without implementing its abstract methods + mocker.patch("source_sendgrid.streams.SendgridStream.__abstractmethods__", set()) + # Mypy yells at us because we're init'ing an abstract class + return SendgridStream() # type: ignore + + +def test_parse_response_gracefully_handles_nulls(mocker, sendgrid_stream: SendgridStream): + response = requests.Response() + mocker.patch.object(response, 'json', return_value=None) + mocker.patch.object(response, 'request', return_value=MagicMock()) + assert [] == list(sendgrid_stream.parse_response(response)) def test_source_wrong_credentials(): From c683fd2d0c903bd5b331f874cfff85b0a27ac74c Mon Sep 17 00:00:00 2001 From: Sherif Nada Date: Mon, 19 Jul 2021 19:54:32 -0700 Subject: [PATCH 6/6] frmat --- .../integration_tests/integration_test.py | 1 + .../source_sendgrid/streams.py | 19 ++++++++++--------- .../source-sendgrid/unit_tests/unit_test.py | 5 +++-- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/airbyte-integrations/connectors/source-sendgrid/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-sendgrid/integration_tests/integration_test.py index 4349a4880fb9..6d275a2544ab 100644 --- a/airbyte-integrations/connectors/source-sendgrid/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/source-sendgrid/integration_tests/integration_test.py @@ -22,5 +22,6 @@ # SOFTWARE. # + def test_example(): assert True diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py index ce2a7b6b84cf..5d8db456ddcb 100644 --- a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py +++ b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py @@ -53,19 +53,20 @@ def parse_response( for record in records: yield record else: - # TODO sendgrid's API is sending null responses at times. This seems like a bug on the API side, so we're adding - # log statements to help reproduce and prevent the connector from failing. - err_msg = f"Response contained no valid JSON data. Response body: {response.text}\n"\ - f"Response status: {response.status_code}\n"\ - f"Response body: {response.text}\n"\ - f"Response headers: {response.headers}\n"\ - f"Request URL: {response.request.url}\n"\ - f"Request body: {response.request.body}\n" + # TODO sendgrid's API is sending null responses at times. This seems like a bug on the API side, so we're adding + # log statements to help reproduce and prevent the connector from failing. + err_msg = ( + f"Response contained no valid JSON data. Response body: {response.text}\n" + f"Response status: {response.status_code}\n" + f"Response body: {response.text}\n" + f"Response headers: {response.headers}\n" + f"Request URL: {response.request.url}\n" + f"Request body: {response.request.body}\n" + ) # do NOT print request headers as it contains auth token self.logger.info(err_msg) - class SendgridStreamOffsetPagination(SendgridStream): offset = 0 diff --git a/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py index 9715b4661054..beaafc9573db 100644 --- a/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py @@ -21,6 +21,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # + from unittest.mock import MagicMock import pytest @@ -40,8 +41,8 @@ def sendgrid_stream_fixture(mocker) -> SendgridStream: def test_parse_response_gracefully_handles_nulls(mocker, sendgrid_stream: SendgridStream): response = requests.Response() - mocker.patch.object(response, 'json', return_value=None) - mocker.patch.object(response, 'request', return_value=MagicMock()) + mocker.patch.object(response, "json", return_value=None) + mocker.patch.object(response, "request", return_value=MagicMock()) assert [] == list(sendgrid_stream.parse_response(response))