From feeee9053088a363874c70e5375ff1c4aebcbb1e Mon Sep 17 00:00:00 2001 From: Christian Gagnon Date: Wed, 15 Sep 2021 10:19:40 -0400 Subject: [PATCH 1/4] Intercom: Fix "updated_at" data type The Intercom API send a integer for the "updated_at" field in the companies endpoint. Which cause issue later on when dbt tries to create the table. --- .../source_intercom/schemas/company_attributes.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json index e5a6d4cc8229..a9b902f1521b 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json @@ -55,8 +55,7 @@ "type": ["null", "boolean"] }, "updated_at": { - "type": ["null", "string"], - "format": "date-time" + "type": ["null", "integer"] } } } From 7ff5c8136738f331f4898ee06d1ccf9b711ca586 Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Mon, 20 Sep 2021 14:05:05 +0300 Subject: [PATCH 2/4] Add start date fromating to timestamp --- .../connectors/source-intercom/source_intercom/source.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) mode change 100644 => 100755 airbyte-integrations/connectors/source-intercom/source_intercom/source.py diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py old mode 100644 new mode 100755 index 479a6da50873..ebae32aaa8de --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -126,13 +126,6 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, record = super().parse_response(response, stream_state, **kwargs) for record in record: - updated_at = record.get(self.cursor_field) - - if updated_at: - record[self.cursor_field] = datetime.fromtimestamp( - record[self.cursor_field] - ).isoformat() # convert timestamp to datetime string - yield from self.filter_by_state(stream_state=stream_state, record=record) def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: @@ -335,6 +328,8 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: AirbyteLogger().log("INFO", f"Using start_date: {config['start_date']}") + config["start_date"] = datetime.strptime(config["start_date"], "%Y-%m-%dT%H:%M:%SZ").timestamp() + auth = TokenAuthenticator(token=config["access_token"]) return [ Admins(authenticator=auth, **config), From 16f1ec5e3847c164640243e1b9ae1dcb46291f51 Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Mon, 20 Sep 2021 14:07:23 +0300 Subject: [PATCH 3/4] Update updated_at field for all schemas and integration tests --- .../integration_tests/abnormal_state.json | 14 +++++++------- .../integration_tests/sample_state.json | 12 ++++++------ .../source_intercom/schemas/companies.json | 3 +-- .../source_intercom/schemas/company_segments.json | 3 +-- .../source_intercom/schemas/contacts.json | 3 +-- .../schemas/conversation_parts.json | 3 +-- .../source_intercom/schemas/conversations.json | 3 +-- .../source_intercom/schemas/segments.json | 3 +-- 8 files changed, 19 insertions(+), 25 deletions(-) mode change 100644 => 100755 airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json mode change 100644 => 100755 airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json mode change 100644 => 100755 airbyte-integrations/connectors/source-intercom/source_intercom/schemas/companies.json mode change 100644 => 100755 airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_segments.json mode change 100644 => 100755 airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json mode change 100644 => 100755 airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversation_parts.json mode change 100644 => 100755 airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json mode change 100644 => 100755 airbyte-integrations/connectors/source-intercom/source_intercom/schemas/segments.json diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json old mode 100644 new mode 100755 index 1bb4ec8dd2e0..b6b4d841531c --- a/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json @@ -1,20 +1,20 @@ { "companies": { - "updated_at": "2022-07-12T10:44:09+00:00" + "updated_at": 7626086649 }, "company_segments": { - "updated_at": "2022-07-12T10:44:09+00:00" + "updated_at": 7626086649 }, "conversations": { - "updated_at": "2022-07-12T10:44:09+00:00" + "updated_at": 7626086649 }, "conversation_parts": { - "updated_at": "2022-07-12T10:44:09+00:00" + "updated_at": 7626086649 }, "contacts": { - "updated_at": "2022-07-12T10:44:09+00:00" + "updated_at": 7626086649 }, "segments": { - "updated_at": "2022-07-12T10:44:09+00:00" + "updated_at": 7626086649 } -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json old mode 100644 new mode 100755 index e3e8395eaea0..e5de1448e423 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json @@ -1,20 +1,20 @@ { "companies": { - "updated_at": "2019-07-12T11:22:46+00:00" + "updated_at": 1626086649 }, "company_segments": { - "updated_at": "2019-07-12T11:22:46+00:00" + "updated_at": 1626086649 }, "conversations": { - "updated_at": "2019-07-12T11:22:46+00:00" + "updated_at": 1626086649 }, "conversation_parts": { - "updated_at": "2019-07-12T11:22:46+00:00" + "updated_at": 1626086649 }, "contacts": { - "updated_at": "2019-07-12T11:22:46+00:00" + "updated_at": 1626086649 }, "segments": { - "updated_at": "2019-07-12T11:22:46+00:00" + "updated_at": 1626086649 } } diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/companies.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/companies.json old mode 100644 new mode 100755 index 0f72dc70babf..5b07257d2951 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/companies.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/companies.json @@ -21,8 +21,7 @@ "type": ["null", "integer"] }, "updated_at": { - "type": ["null", "string"], - "format": "date-time" + "type": ["null", "integer"] }, "monthly_spend": { "type": ["null", "number"], diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_segments.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_segments.json old mode 100644 new mode 100755 index 67f1f02387ca..ca08be8a7923 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_segments.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_segments.json @@ -21,8 +21,7 @@ "type": ["null", "string"] }, "updated_at": { - "type": ["null", "string"], - "format": "date-time" + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json old mode 100644 new mode 100755 index 38944193a20c..f343927b3661 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json @@ -72,8 +72,7 @@ "type": ["null", "integer"] }, "updated_at": { - "type": ["null", "string"], - "format": "date-time" + "type": ["null", "integer"] }, "signed_up_at": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversation_parts.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversation_parts.json old mode 100644 new mode 100755 index 8b5ad38dc3f5..10621c781d2f --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversation_parts.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversation_parts.json @@ -113,8 +113,7 @@ "type": ["null", "string"] }, "updated_at": { - "type": ["null", "string"], - "format": "date-time" + "type": ["null", "integer"] }, "redacted": { "type": ["null", "boolean"] diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json old mode 100644 new mode 100755 index 68ae977e9ceb..2be8a0eb19f4 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json @@ -416,8 +416,7 @@ "type": ["null", "string"] }, "updated_at": { - "type": ["null", "string"], - "format": "date-time" + "type": ["null", "integer"] }, "user": { "type": ["null", "object"], diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/segments.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/segments.json old mode 100644 new mode 100755 index 67f1f02387ca..ca08be8a7923 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/segments.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/segments.json @@ -21,8 +21,7 @@ "type": ["null", "string"] }, "updated_at": { - "type": ["null", "string"], - "format": "date-time" + "type": ["null", "integer"] } } } From 104fbbde72e8e8299e960f1e9ea859bbb1ef0c12 Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Mon, 20 Sep 2021 14:29:26 +0300 Subject: [PATCH 4/4] Bumped version --- .../d8313939-3782-41b0-be29-b3ca20d8dd3a.json | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-integrations/connectors/source-intercom/Dockerfile | 2 +- docs/integrations/sources/intercom.md | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json index e99511fbad0a..a536e049b791 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "d8313939-3782-41b0-be29-b3ca20d8dd3a", "name": "Intercom", "dockerRepository": "airbyte/source-intercom", - "dockerImageTag": "0.1.3", + "dockerImageTag": "0.1.4", "documentationUrl": "https://docs.airbyte.io/integrations/sources/intercom", "icon": "intercom.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 2279ca821191..c904054b512d 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -200,7 +200,7 @@ - sourceDefinitionId: d8313939-3782-41b0-be29-b3ca20d8dd3a name: Intercom dockerRepository: airbyte/source-intercom - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/sources/intercom icon: intercom.svg - sourceDefinitionId: 68e63de2-bb83-4c7e-93fa-a8a9051e3993 diff --git a/airbyte-integrations/connectors/source-intercom/Dockerfile b/airbyte-integrations/connectors/source-intercom/Dockerfile index 0a60fd900fe7..50dd9d320d14 100644 --- a/airbyte-integrations/connectors/source-intercom/Dockerfile +++ b/airbyte-integrations/connectors/source-intercom/Dockerfile @@ -34,5 +34,5 @@ COPY source_intercom ./source_intercom ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/source-intercom diff --git a/docs/integrations/sources/intercom.md b/docs/integrations/sources/intercom.md index 76bc9cbd4d0c..5bf35b209ed2 100644 --- a/docs/integrations/sources/intercom.md +++ b/docs/integrations/sources/intercom.md @@ -55,6 +55,7 @@ Please read [How to get your Access Token](https://developers.intercom.com/build | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | +| 0.1.4 | 2021-09-20 | [6087](https://github.com/airbytehq/airbyte/pull/6087) | Corrected updated_at field data type in schemas | | 0.1.3 | 2021-09-08 | [5908](https://github.com/airbytehq/airbyte/pull/5908) | Corrected timestamp and arrays in schemas | | 0.1.2 | 2021-08-19 | [5531](https://github.com/airbytehq/airbyte/pull/5531) | Corrected pagination | | 0.1.1 | 2021-07-31 | [5123](https://github.com/airbytehq/airbyte/pull/5123) | Corrected rate limit |