🐛 Source Sendgrid: add start_time config and correct primary_key (#4682)

* add start_time config and correct primary_key * correct integration tests * correct type * config txt and primary_key
airbytehq · Jul 22, 2021 · f19f5d7 · f19f5d7
1 parent 551038b
commit f19f5d7
Show file tree

Hide file tree

Showing 8 changed files with 23 additions and 63 deletions.
diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87.json
@@ -2,7 +2,7 @@
   "sourceDefinitionId": "fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87",
   "name": "Sendgrid",
   "dockerRepository": "airbyte/source-sendgrid",
-  "dockerImageTag": "0.2.4",
+  "dockerImageTag": "0.2.5",
   "documentationUrl": "https://hub.docker.com/r/airbyte/source-sendgrid",
   "icon": "sendgrid.svg"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -68,7 +68,7 @@
 - sourceDefinitionId: fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87
   name: Sendgrid
   dockerRepository: airbyte/source-sendgrid
-  dockerImageTag: 0.2.4
+  dockerImageTag: 0.2.5
   documentationUrl: https://hub.docker.com/r/airbyte/source-sendgrid
   icon: sendgrid.svg
 - sourceDefinitionId: 9e0556f4-69df-4522-a3fb-03264d36b348

diff --git a/airbyte-integrations/connectors/source-sendgrid/Dockerfile b/airbyte-integrations/connectors/source-sendgrid/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.4
+LABEL io.airbyte.version=0.2.5
 LABEL io.airbyte.name=airbyte/source-sendgrid
diff --git a/...egrations/connectors/source-sendgrid/sample_files/no_spam_reports_configured_catalog.json b/...egrations/connectors/source-sendgrid/sample_files/no_spam_reports_configured_catalog.json
@@ -354,35 +354,6 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
     },
-    {
-      "stream": {
-        "name": "blocks",
-        "json_schema": {
-          "$schema": "http://json-schema.org/draft-07/schema#",
-          "type": "object",
-          "properties": {
-            "created": {
-              "type": "integer"
-            },
-            "email": {
-              "type": "string"
-            },
-            "reason": {
-              "type": "string"
-            },
-            "status": {
-              "type": "string"
-            }
-          }
-        },
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["created"]
-      },
-      "sync_mode": "incremental",
-      "cursor_field": ["created"],
-      "destination_sync_mode": "append"
-    },
     {
       "stream": {
         "name": "bounces",
@@ -411,32 +382,6 @@
       "sync_mode": "incremental",
       "cursor_field": ["created"],
       "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "invalid_emails",
-        "json_schema": {
-          "$schema": "http://json-schema.org/draft-07/schema#",
-          "type": "object",
-          "properties": {
-            "created": {
-              "type": "integer"
-            },
-            "email": {
-              "type": "string"
-            },
-            "reason": {
-              "type": "string"
-            }
-          }
-        },
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["created"]
-      },
-      "sync_mode": "incremental",
-      "cursor_field": ["created"],
-      "destination_sync_mode": "append"
     }
   ]
 }
diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/schemas/contacts.json b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/schemas/contacts.json
@@ -9,7 +9,7 @@
       "type": "string"
     },
     "alternate_emails": {
-      "type": "array"
+      "type": ["null", "array"]
     },
     "city": {
       "type": "string"
@@ -36,13 +36,13 @@
       "type": "string"
     },
     "list_ids": {
-      "type": "array"
+      "type": ["null", "array"]
     },
     "created_at": {
-      "type": "array"
+      "type": "string"
     },
     "updated_at": {
-      "type": "array"
+      "type": "string"
     },
     "_metadata": {
       "type": "object",

diff --git a/...e-integrations/connectors/source-sendgrid/source_sendgrid/schemas/suppression_groups.json b/...e-integrations/connectors/source-sendgrid/source_sendgrid/schemas/suppression_groups.json
@@ -3,7 +3,7 @@
   "type": "object",
   "properties": {
     "id": {
-      "type": "string"
+      "type": "integer"
     },
     "name": {
       "type": "string"

diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json
@@ -10,6 +10,11 @@
       "apikey": {
         "type": "string",
         "description": "API Key, use <a href=\"https://app.sendgrid.com/settings/api_keys/\">admin</a> to generate this key."
+      },
+      "start_time": {
+        "type": "integer",
+        "description": "Start time in timestamp integer format. Any data before this timestamp will not be replicated.",
+        "examples": [1558359837]
       }
     }
   }

diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py
@@ -190,6 +190,8 @@ def initial_path() -> str:
 
 
 class GlobalSuppressions(SendgridStreamOffsetPagination, SendgridStreamIncrementalMixin):
+    primary_key = "email"
+
     def path(self, **kwargs) -> str:
         return "suppression/unsubscribes"
 
@@ -205,20 +207,28 @@ def path(self, **kwargs) -> str:
 
 
 class Blocks(SendgridStreamOffsetPagination, SendgridStreamIncrementalMixin):
+    primary_key = "email"
+
     def path(self, **kwargs) -> str:
         return "suppression/blocks"
 
 
 class Bounces(SendgridStream, SendgridStreamIncrementalMixin):
+    primary_key = "email"
+
     def path(self, **kwargs) -> str:
         return "suppression/bounces"
 
 
 class InvalidEmails(SendgridStreamOffsetPagination, SendgridStreamIncrementalMixin):
+    primary_key = "email"
+
     def path(self, **kwargs) -> str:
         return "suppression/invalid_emails"
 
 
 class SpamReports(SendgridStreamOffsetPagination, SendgridStreamIncrementalMixin):
+    primary_key = "email"
+
     def path(self, **kwargs) -> str:
         return "suppression/spam_reports"