From 28a016fa8336ca1cb088fa9354572a4ddd61b39b Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Sat, 10 Jul 2021 15:37:10 -0300 Subject: [PATCH 1/4] add start_time config and correct primary_key --- .../fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87.json | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-integrations/connectors/source-sendgrid/Dockerfile | 2 +- .../connectors/source-sendgrid/source_sendgrid/spec.json | 4 ++++ .../connectors/source-sendgrid/source_sendgrid/streams.py | 2 +- 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87.json index 7cd3166d0ba6..19492bfa75fd 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87", "name": "Sendgrid", "dockerRepository": "airbyte/source-sendgrid", - "dockerImageTag": "0.2.4", + "dockerImageTag": "0.2.5", "documentationUrl": "https://hub.docker.com/r/airbyte/source-sendgrid", "icon": "sendgrid.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 0ab6f14de2e0..cb0b26003caa 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -68,7 +68,7 @@ - sourceDefinitionId: fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87 name: Sendgrid dockerRepository: airbyte/source-sendgrid - dockerImageTag: 0.2.4 + dockerImageTag: 0.2.5 documentationUrl: https://hub.docker.com/r/airbyte/source-sendgrid icon: sendgrid.svg - sourceDefinitionId: 9e0556f4-69df-4522-a3fb-03264d36b348 diff --git a/airbyte-integrations/connectors/source-sendgrid/Dockerfile b/airbyte-integrations/connectors/source-sendgrid/Dockerfile index 51bad6728ebe..6e7a58ae1537 100644 --- a/airbyte-integrations/connectors/source-sendgrid/Dockerfile +++ b/airbyte-integrations/connectors/source-sendgrid/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.4 +LABEL io.airbyte.version=0.2.5 LABEL io.airbyte.name=airbyte/source-sendgrid diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json index 1fe5e486324c..cb30232ec6b5 100644 --- a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json +++ b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json @@ -10,6 +10,10 @@ "apikey": { "type": "string", "description": "API Key, use admin to generate this key." + }, + "start_time": { + "type": "integer", + "description": "Start time in timestamp integer format, example: 1558359837" } } } diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py index a30185c3c7d1..f3f97908b943 100644 --- a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py +++ b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py @@ -32,7 +32,7 @@ class SendgridStream(HttpStream, ABC): url_base = "https://api.sendgrid.com/v3/" - primary_key = "id" + primary_key = "email" limit = 50 data_field = None From d91679cc1c10e563ff70a222e95ca22b49177ca7 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Sat, 10 Jul 2021 16:15:43 -0300 Subject: [PATCH 2/4] correct integration tests --- .../no_spam_reports_configured_catalog.json | 55 ------------------- .../source_sendgrid/schemas/contacts.json | 4 +- .../schemas/suppression_groups.json | 2 +- 3 files changed, 3 insertions(+), 58 deletions(-) diff --git a/airbyte-integrations/connectors/source-sendgrid/sample_files/no_spam_reports_configured_catalog.json b/airbyte-integrations/connectors/source-sendgrid/sample_files/no_spam_reports_configured_catalog.json index 07913b6d2c71..821b3ceaf169 100644 --- a/airbyte-integrations/connectors/source-sendgrid/sample_files/no_spam_reports_configured_catalog.json +++ b/airbyte-integrations/connectors/source-sendgrid/sample_files/no_spam_reports_configured_catalog.json @@ -354,35 +354,6 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, - { - "stream": { - "name": "blocks", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "created": { - "type": "integer" - }, - "email": { - "type": "string" - }, - "reason": { - "type": "string" - }, - "status": { - "type": "string" - } - } - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"] - }, - "sync_mode": "incremental", - "cursor_field": ["created"], - "destination_sync_mode": "append" - }, { "stream": { "name": "bounces", @@ -411,32 +382,6 @@ "sync_mode": "incremental", "cursor_field": ["created"], "destination_sync_mode": "append" - }, - { - "stream": { - "name": "invalid_emails", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "created": { - "type": "integer" - }, - "email": { - "type": "string" - }, - "reason": { - "type": "string" - } - } - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"] - }, - "sync_mode": "incremental", - "cursor_field": ["created"], - "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/schemas/contacts.json b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/schemas/contacts.json index e48556c344de..8cfcaf31e02f 100644 --- a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/schemas/contacts.json @@ -39,10 +39,10 @@ "type": "array" }, "created_at": { - "type": "array" + "type": "integer" }, "updated_at": { - "type": "array" + "type": "integer" }, "_metadata": { "type": "object", diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/schemas/suppression_groups.json b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/schemas/suppression_groups.json index 4f917207a4ab..f9ebb4113fda 100644 --- a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/schemas/suppression_groups.json +++ b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/schemas/suppression_groups.json @@ -3,7 +3,7 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": "integer" }, "name": { "type": "string" From 918518a97e0b35475e15d590e2a8788973f6c783 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Sat, 10 Jul 2021 16:29:16 -0300 Subject: [PATCH 3/4] correct type --- .../source-sendgrid/source_sendgrid/schemas/contacts.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/schemas/contacts.json b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/schemas/contacts.json index 8cfcaf31e02f..b30832672ab5 100644 --- a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/schemas/contacts.json @@ -9,7 +9,7 @@ "type": "string" }, "alternate_emails": { - "type": "array" + "type": ["null", "array"] }, "city": { "type": "string" @@ -36,13 +36,13 @@ "type": "string" }, "list_ids": { - "type": "array" + "type": ["null", "array"] }, "created_at": { - "type": "integer" + "type": "string" }, "updated_at": { - "type": "integer" + "type": "string" }, "_metadata": { "type": "object", From 9e2320e74884a3aff5f8dbbeb261ec346016593f Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Sat, 10 Jul 2021 16:42:18 -0300 Subject: [PATCH 4/4] config txt and primary_key --- .../source-sendgrid/source_sendgrid/spec.json | 3 ++- .../source-sendgrid/source_sendgrid/streams.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json index cb30232ec6b5..664cde06193d 100644 --- a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json +++ b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json @@ -13,7 +13,8 @@ }, "start_time": { "type": "integer", - "description": "Start time in timestamp integer format, example: 1558359837" + "description": "Start time in timestamp integer format. Any data before this timestamp will not be replicated.", + "examples": [1558359837] } } } diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py index f3f97908b943..c8c2a7de1b16 100644 --- a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py +++ b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py @@ -32,7 +32,7 @@ class SendgridStream(HttpStream, ABC): url_base = "https://api.sendgrid.com/v3/" - primary_key = "email" + primary_key = "id" limit = 50 data_field = None @@ -190,6 +190,8 @@ def initial_path() -> str: class GlobalSuppressions(SendgridStreamOffsetPagination, SendgridStreamIncrementalMixin): + primary_key = "email" + def path(self, **kwargs) -> str: return "suppression/unsubscribes" @@ -205,20 +207,28 @@ def path(self, **kwargs) -> str: class Blocks(SendgridStreamOffsetPagination, SendgridStreamIncrementalMixin): + primary_key = "email" + def path(self, **kwargs) -> str: return "suppression/blocks" class Bounces(SendgridStream, SendgridStreamIncrementalMixin): + primary_key = "email" + def path(self, **kwargs) -> str: return "suppression/bounces" class InvalidEmails(SendgridStreamOffsetPagination, SendgridStreamIncrementalMixin): + primary_key = "email" + def path(self, **kwargs) -> str: return "suppression/invalid_emails" class SpamReports(SendgridStreamOffsetPagination, SendgridStreamIncrementalMixin): + primary_key = "email" + def path(self, **kwargs) -> str: return "suppression/spam_reports"