From 3c3d8503668614e283b50549d0c8e3b25923dfa8 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Sat, 10 Jul 2021 15:56:28 -0300 Subject: [PATCH 1/9] rename float_ts to ts cursor_field --- .../source-slack/integration_tests/configured_catalog.json | 4 ++-- .../connectors/source-slack/source_slack/source.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-slack/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-slack/integration_tests/configured_catalog.json index f26dce72388f..997fca524786 100644 --- a/airbyte-integrations/connectors/source-slack/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-slack/integration_tests/configured_catalog.json @@ -215,9 +215,9 @@ }, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["float_ts"] + "default_cursor_field": ["ts"] }, - "cursor_field": ["float_ts"], + "cursor_field": ["ts"], "sync_mode": "incremental", "destination_sync_mode": "append" } diff --git a/airbyte-integrations/connectors/source-slack/source_slack/source.py b/airbyte-integrations/connectors/source-slack/source_slack/source.py index 7907d718e907..50eefb14dd82 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/source.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/source.py @@ -142,7 +142,7 @@ def chunk_date_range(start_date: DateTime, interval=pendulum.duration(days=1)) - class IncrementalMessageStream(SlackStream, ABC): data_field = "messages" - cursor_field = "float_ts" + cursor_field = "ts" primary_key = ["channel_id", "ts"] def __init__(self, default_start_date: DateTime, **kwargs): From a8156f471d491d27b6171f1d5217a578aabec27d Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Sat, 10 Jul 2021 17:11:17 -0300 Subject: [PATCH 2/9] add float_ts --- .../integration_tests/configured_catalog.json | 7 +++++-- .../source-slack/sample_files/configured_catalog.json | 9 ++++++--- .../source_slack/schemas/channel_messages.json | 3 +++ .../source-slack/source_slack/schemas/threads.json | 5 ++++- .../connectors/source-slack/source_slack/source.py | 2 +- 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-slack/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-slack/integration_tests/configured_catalog.json index 997fca524786..0246a42f572b 100644 --- a/airbyte-integrations/connectors/source-slack/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-slack/integration_tests/configured_catalog.json @@ -192,6 +192,9 @@ "ts": { "type": ["null", "string"] }, + "float_ts": { + "type": ["null", "string"] + }, "type": { "type": ["null", "string"] }, @@ -215,9 +218,9 @@ }, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["ts"] + "default_cursor_field": ["float_ts"] }, - "cursor_field": ["ts"], + "cursor_field": ["float_ts"], "sync_mode": "incremental", "destination_sync_mode": "append" } diff --git a/airbyte-integrations/connectors/source-slack/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-slack/sample_files/configured_catalog.json index b3dfd2074f0d..cf15dcd36fb6 100644 --- a/airbyte-integrations/connectors/source-slack/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-slack/sample_files/configured_catalog.json @@ -24,7 +24,10 @@ "type": ["null", "string"] }, "ts": { - "type": ["null", "number"] + "type": ["null", "string"] + }, + "float_ts": { + "type": ["null", "string"] }, "team": { "type": ["null", "string"] @@ -66,9 +69,9 @@ }, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["ts"] + "default_cursor_field": ["float_ts"] }, - "cursor_field": ["ts"], + "cursor_field": ["float_ts"], "sync_mode": "incremental", "destination_sync_mode": "append" } diff --git a/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json b/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json index 806613ad6ab0..3af6bd587be2 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json +++ b/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json @@ -186,6 +186,9 @@ "ts": { "type": ["null", "string"] }, + "float_ts": { + "type": ["null", "string"] + }, "type": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json b/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json index eeed705e2c2f..310144b2bc36 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json +++ b/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json @@ -18,7 +18,10 @@ "type": ["null", "string"] }, "ts": { - "type": ["null", "number"] + "type": ["null", "string"] + }, + "float_ts": { + "type": ["null", "string"] }, "team": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-slack/source_slack/source.py b/airbyte-integrations/connectors/source-slack/source_slack/source.py index 50eefb14dd82..7907d718e907 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/source.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/source.py @@ -142,7 +142,7 @@ def chunk_date_range(start_date: DateTime, interval=pendulum.duration(days=1)) - class IncrementalMessageStream(SlackStream, ABC): data_field = "messages" - cursor_field = "ts" + cursor_field = "float_ts" primary_key = ["channel_id", "ts"] def __init__(self, default_start_date: DateTime, **kwargs): From bd022042f12e21c4d015fbcc58ecd1ebb5713716 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Sat, 10 Jul 2021 17:13:56 -0300 Subject: [PATCH 3/9] change float_ts to number --- .../source-slack/integration_tests/configured_catalog.json | 2 +- .../source-slack/sample_files/configured_catalog.json | 2 +- .../connectors/source-slack/source_slack/schemas/threads.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-slack/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-slack/integration_tests/configured_catalog.json index 0246a42f572b..c03b16c51c09 100644 --- a/airbyte-integrations/connectors/source-slack/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-slack/integration_tests/configured_catalog.json @@ -193,7 +193,7 @@ "type": ["null", "string"] }, "float_ts": { - "type": ["null", "string"] + "type": ["null", "number"] }, "type": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-slack/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-slack/sample_files/configured_catalog.json index cf15dcd36fb6..7cc0909e504f 100644 --- a/airbyte-integrations/connectors/source-slack/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-slack/sample_files/configured_catalog.json @@ -27,7 +27,7 @@ "type": ["null", "string"] }, "float_ts": { - "type": ["null", "string"] + "type": ["null", "number"] }, "team": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json b/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json index 310144b2bc36..d7804390f839 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json +++ b/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json @@ -21,7 +21,7 @@ "type": ["null", "string"] }, "float_ts": { - "type": ["null", "string"] + "type": ["null", "number"] }, "team": { "type": ["null", "string"] From a29df10b34ebcf1cc1baffe1b75e1970643b72f4 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Sat, 10 Jul 2021 17:15:38 -0300 Subject: [PATCH 4/9] change channel_msg --- .../source-slack/source_slack/schemas/channel_messages.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json b/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json index 3af6bd587be2..5b94bf0c604d 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json +++ b/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json @@ -187,7 +187,7 @@ "type": ["null", "string"] }, "float_ts": { - "type": ["null", "string"] + "type": ["null", "number"] }, "type": { "type": ["null", "string"] From 0e3c3f76c84c3261c86026fc603f7b6e4ef55552 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Sat, 10 Jul 2021 17:33:21 -0300 Subject: [PATCH 5/9] bump version --- .../c2281cee-86f9-4a86-bb48-d23286b4c7bd.json | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-integrations/connectors/source-slack/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c2281cee-86f9-4a86-bb48-d23286b4c7bd.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c2281cee-86f9-4a86-bb48-d23286b4c7bd.json index 75e5f20c283a..d535b0f62aa6 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c2281cee-86f9-4a86-bb48-d23286b4c7bd.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c2281cee-86f9-4a86-bb48-d23286b4c7bd.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "c2281cee-86f9-4a86-bb48-d23286b4c7bd", "name": "Slack", "dockerRepository": "airbyte/source-slack", - "dockerImageTag": "0.1.7", + "dockerImageTag": "0.1.8", "documentationUrl": "https://hub.docker.com/repository/docker/airbyte/source-slack", "icon": "slack.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 0ab6f14de2e0..acb58b315671 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -335,7 +335,7 @@ - sourceDefinitionId: c2281cee-86f9-4a86-bb48-d23286b4c7bd name: Slack dockerRepository: airbyte/source-slack - dockerImageTag: 0.1.7 + dockerImageTag: 0.1.8 documentationUrl: https://hub.docker.com/repository/docker/airbyte/source-slack icon: slack.svg - sourceDefinitionId: 6ff047c0-f5d5-4ce5-8c81-204a830fa7e1 diff --git a/airbyte-integrations/connectors/source-slack/Dockerfile b/airbyte-integrations/connectors/source-slack/Dockerfile index 061d046f0839..745fecf685db 100644 --- a/airbyte-integrations/connectors/source-slack/Dockerfile +++ b/airbyte-integrations/connectors/source-slack/Dockerfile @@ -16,5 +16,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/source-slack From bedfe3b81856a09b7c4229d8120fe6c6ce25d74c Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Wed, 14 Jul 2021 10:50:42 -0300 Subject: [PATCH 6/9] increase default timeout_seconds slack acc test --- .../connectors/source-slack/acceptance-test-config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml index a106a054c13c..54381c1aac0d 100644 --- a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml @@ -16,6 +16,7 @@ tests: incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 3600 cursor_paths: channel_messages: ["float_ts"] full_refresh: From 6271818484cb90e854718b035ac742ac971c3a3e Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Wed, 14 Jul 2021 14:34:14 -0300 Subject: [PATCH 7/9] timeout_seconds to 1750 --- .../connectors/source-slack/acceptance-test-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml index 54381c1aac0d..0d8b315aa6e3 100644 --- a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml @@ -16,7 +16,7 @@ tests: incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 3600 + timeout_seconds: 1750 cursor_paths: channel_messages: ["float_ts"] full_refresh: From 25e30831c62339e81293b0afb194e775e6a183a3 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Wed, 14 Jul 2021 15:33:50 -0300 Subject: [PATCH 8/9] timeout_seconds to 3600 :p --- .../connectors/source-slack/acceptance-test-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml index 0d8b315aa6e3..54381c1aac0d 100644 --- a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml @@ -16,7 +16,7 @@ tests: incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 1750 + timeout_seconds: 3600 cursor_paths: channel_messages: ["float_ts"] full_refresh: From ab2173c87a7243b71f44b04308d56aee20c15319 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Wed, 14 Jul 2021 17:45:53 -0300 Subject: [PATCH 9/9] add changelog for slack connector --- docs/integrations/sources/slack.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/integrations/sources/slack.md b/docs/integrations/sources/slack.md index 5cc83607d8ed..52e4277d1d29 100644 --- a/docs/integrations/sources/slack.md +++ b/docs/integrations/sources/slack.md @@ -97,3 +97,9 @@ You can no longer create "Legacy" API Keys, but if you already have one, you can We recommend creating a restricted, read-only key specifically for Airbyte access. This will allow you to control which resources Airbyte should be able to access. +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :-------- | :----- | :------ | +| 0.1.8 | 2021-07-14 | [4683](https://github.com/airbytehq/airbyte/pull/4683) | Add float_ts primary key | +| 0.1.7 | 2021-06-25 | [3978](https://github.com/airbytehq/airbyte/pull/3978) | Release Slack CDK Connector |