From 566fc1ce96f1db4b828278eaef787646f53853ad Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Tue, 4 Oct 2022 17:05:48 +0300 Subject: [PATCH 1/3] #17506 Source Slack, Source Tiktok-marketing - fix builds --- .../source-slack/acceptance-test-config.yml | 2 - .../integration_tests/expected_records.txt | 35 ------------ .../source-tiktok-marketing/Dockerfile | 2 +- .../integration_tests/streams_basic.json | 18 ++++-- .../streams_reports_daily.json | 56 +++++++++++++------ .../streams_reports_lifetime.json | 35 ++++++++---- .../streams_with_day_granularity.json | 56 +++++++++++++------ .../streams_with_lifetime_granularity.json | 35 ++++++++---- .../source_tiktok_marketing/streams.py | 3 +- .../unit_tests/unit_test.py | 5 +- docs/integrations/sources/tiktok-marketing.md | 3 +- 11 files changed, 150 insertions(+), 100 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-slack/integration_tests/expected_records.txt diff --git a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml index a222ae64692d..76cc642d536d 100644 --- a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml @@ -18,8 +18,6 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/full_refresh_catalog.json" - expect_records: - path: "integration_tests/expected_records.txt" incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/incremental_catalog.json" diff --git a/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.txt deleted file mode 100644 index 23d1bbd4a234..000000000000 --- a/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.txt +++ /dev/null @@ -1,35 +0,0 @@ -{"stream": "users", "data": {"id": "USLACKBOT", "team_id": "T01AB4DDR2N", "name": "slackbot", "deleted": false, "color": "757575", "real_name": "Slackbot", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Slackbot", "real_name_normalized": "Slackbot", "display_name": "Slackbot", "display_name_normalized": "Slackbot", "fields": {}, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "sv41d8cd98f0", "always_active": true, "first_name": "slackbot", "last_name": "", "image_24": "https://a.slack-edge.com/80588/img/slackbot_24.png", "image_32": "https://a.slack-edge.com/80588/img/slackbot_32.png", "image_48": "https://a.slack-edge.com/80588/img/slackbot_48.png", "image_72": "https://a.slack-edge.com/80588/img/slackbot_72.png", "image_192": "https://a.slack-edge.com/80588/marketing/img/avatars/slackbot/avatar-slackbot.png", "image_512": "https://a.slack-edge.com/80588/img/slackbot_512.png", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 0, "is_email_confirmed": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397747} -{"stream": "users", "data": {"id": "U019WF8P78X", "team_id": "T01AB4DDR2N", "name": "google_calendar", "deleted": false, "color": "3c989f", "real_name": "Google Calendar", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Google Calendar", "real_name_normalized": "Google Calendar", "display_name": "", "display_name_normalized": "", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "27bf72bdda29", "api_app_id": "ADZ494LHY", "always_active": true, "image_original": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_original.png", "is_custom_image": true, "bot_id": "B01APS4667K", "first_name": "Google", "last_name": "Calendar", "image_24": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_24.png", "image_32": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_32.png", "image_48": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_48.png", "image_72": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_72.png", "image_192": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_192.png", "image_512": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_512.png", "image_1024": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_1024.png", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": true, "is_app_user": false, "updated": 1622570515, "is_email_confirmed": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397751} -{"stream": "users", "data": {"id": "U019WFB2G31", "team_id": "T01AB4DDR2N", "name": "simplepoll", "deleted": false, "color": "674b1b", "real_name": "Simple Poll", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Simple Poll", "real_name_normalized": "Simple Poll", "display_name": "", "display_name_normalized": "", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "0eee8341dcfc", "api_app_id": "A0HFW7MR6", "always_active": true, "image_original": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_original.png", "is_custom_image": true, "bot_id": "B01ABDTVBJ7", "first_name": "Simple", "last_name": "Poll", "image_24": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_24.png", "image_32": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_32.png", "image_48": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_48.png", "image_72": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_72.png", "image_192": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_192.png", "image_512": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_512.png", "image_1024": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_1024.png", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": true, "is_app_user": false, "updated": 1661533967, "is_email_confirmed": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397752} -{"stream": "users", "data": {"id": "U01A6TEQXRU", "team_id": "T01AB4DDR2N", "name": "sherif", "deleted": false, "color": "5b89d5", "real_name": "sherif (please use other account)", "tz": "Asia/Beirut", "tz_label": "Eastern European Summer Time", "tz_offset": 10800, "profile": {"title": "Connectors Engineering", "phone": "", "skype": "", "real_name": "sherif (please use other account)", "real_name_normalized": "sherif (please use other account)", "display_name": "s", "display_name_normalized": "s", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "6613ce572ac2", "image_original": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_original.jpg", "is_custom_image": true, "email": "sherif@airbyte.io", "huddle_state": "default_unset", "first_name": "sherif", "last_name": "(please use other account)", "image_24": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_24.jpg", "image_32": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_32.jpg", "image_48": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_48.jpg", "image_72": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_72.jpg", "image_192": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_192.jpg", "image_512": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_512.jpg", "image_1024": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_1024.jpg", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": true, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1664337135, "is_email_confirmed": true, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397753} -{"stream": "users", "data": {"id": "U01A81VHD45", "team_id": "T01AB4DDR2N", "name": "michel", "deleted": false, "color": "9f69e7", "real_name": "Michel Tricot", "tz": "Europe/Brussels", "tz_label": "Central European Summer Time", "tz_offset": 7200, "profile": {"title": "Co-Founder & CEO", "phone": "", "skype": "", "real_name": "Michel Tricot", "real_name_normalized": "Michel Tricot", "display_name": "Michel", "display_name_normalized": "Michel", "fields": null, "status_text": "", "status_emoji": ":airbyte:", "status_emoji_display_info": [{"emoji_name": "airbyte", "display_url": "https://emoji.slack-edge.com/T01AB4DDR2N/airbyte/2ba84475f2e46e08.png"}], "status_expiration": 0, "avatar_hash": "9cec7ecdfa05", "image_original": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_original.jpg", "is_custom_image": true, "email": "michel@airbyte.io", "huddle_state": "default_unset", "huddle_state_expiration_ts": 0, "first_name": "Michel", "last_name": "Tricot", "image_24": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_24.jpg", "image_32": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_32.jpg", "image_48": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_48.jpg", "image_72": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_72.jpg", "image_192": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_192.jpg", "image_512": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_512.jpg", "image_1024": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_1024.jpg", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": true, "is_owner": true, "is_primary_owner": true, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1664190286, "is_email_confirmed": true, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397754} -{"stream": "users", "data": {"id": "U01AB6V6NMQ", "team_id": "T01AB4DDR2N", "name": "john", "deleted": false, "color": "4bbe2e", "real_name": "John Lafleur", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "Co-founder", "phone": "4153597503", "skype": "", "real_name": "John Lafleur", "real_name_normalized": "John Lafleur", "display_name": "John (Airbyte)", "display_name_normalized": "John (Airbyte)", "fields": null, "status_text": "", "status_emoji": ":airbyte:", "status_emoji_display_info": [{"emoji_name": "airbyte", "display_url": "https://emoji.slack-edge.com/T01AB4DDR2N/airbyte/2ba84475f2e46e08.png"}], "status_expiration": 0, "avatar_hash": "6a64a9108cd2", "image_original": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_original.jpg", "is_custom_image": true, "email": "john@airbyte.io", "huddle_state": "default_unset", "huddle_state_expiration_ts": 0, "first_name": "John", "last_name": "Lafleur", "image_24": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_24.jpg", "image_32": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_32.jpg", "image_48": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_48.jpg", "image_72": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_72.jpg", "image_192": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_192.jpg", "image_512": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_512.jpg", "image_1024": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_1024.jpg", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": true, "is_owner": true, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1663767942, "is_email_confirmed": true, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397755} -{"stream": "users", "data": {"id": "U01ABDY04RZ", "team_id": "T01AB4DDR2N", "name": "github-x", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "GitHub (Legacy)", "real_name_normalized": "GitHub (Legacy)", "display_name": "", "display_name_normalized": "", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "de0095e0c631", "api_app_id": "A8GBNUWU8", "always_active": true, "image_original": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_original.png", "is_custom_image": true, "bot_id": "B01A85CCXT7", "first_name": "GitHub", "last_name": "(Legacy)", "image_24": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_24.png", "image_32": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_32.png", "image_48": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_48.png", "image_72": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_72.png", "image_192": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_192.png", "image_512": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_512.png", "image_1024": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_1024.png", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_bot": true, "is_app_user": true, "updated": 1618854795}, "emitted_at": 1661589397756} -{"stream": "users", "data": {"id": "U01ADKHBJKC", "team_id": "T01AB4DDR2N", "name": "jared", "deleted": false, "color": "e0a729", "real_name": "Jared (message me on other account)", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "Engineering (Platform) @ Airbyte", "phone": "", "skype": "", "real_name": "Jared (message me on other account)", "real_name_normalized": "Jared (message me on other account)", "display_name": "Jared Rhizor (Airbyte)", "display_name_normalized": "Jared Rhizor (Airbyte)", "fields": null, "status_text": "", "status_emoji": ":airbyte:", "status_emoji_display_info": [{"emoji_name": "airbyte", "display_url": "https://emoji.slack-edge.com/T01AB4DDR2N/airbyte/2ba84475f2e46e08.png"}], "status_expiration": 0, "avatar_hash": "438b7f2a6e75", "image_original": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_original.jpg", "is_custom_image": true, "email": "jared@airbyte.io", "first_name": "Jared", "last_name": "(message me on other account)", "image_24": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_24.jpg", "image_32": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_32.jpg", "image_48": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_48.jpg", "image_72": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_72.jpg", "image_192": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_192.jpg", "image_512": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_512.jpg", "image_1024": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_1024.jpg", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": true, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1662549241, "is_email_confirmed": true, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397757} -{"stream": "users", "data": {"id": "U01ADQ3FGMC", "team_id": "T01AB4DDR2N", "name": "zapier", "deleted": false, "color": "df3dc0", "real_name": "Zapier", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Zapier", "real_name_normalized": "Zapier", "display_name": "", "display_name_normalized": "", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "302ec95148aa", "api_app_id": "A024R9PQM", "always_active": true, "image_original": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_original.png", "is_custom_image": true, "bot_id": "B01ALN9M9GT", "first_name": "Zapier", "last_name": "", "image_24": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_24.png", "image_32": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_32.png", "image_48": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_48.png", "image_72": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_72.png", "image_192": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_192.png", "image_512": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_512.png", "image_1024": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_1024.png", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": true, "is_app_user": false, "updated": 1662549246, "is_email_confirmed": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397758} -{"stream": "users", "data": {"id": "U01ADSJ1D5H", "team_id": "T01AB4DDR2N", "name": "jamakase54", "deleted": false, "color": "2b6836", "real_name": "Artem Astapenko", "tz": "Europe/Moscow", "tz_label": "Moscow Time", "tz_offset": 10800, "profile": {"title": "CEO and Tech Lead @ Jamakase Technologies", "phone": "", "skype": "", "real_name": "Artem Astapenko", "real_name_normalized": "Artem Astapenko", "display_name": "Artem Astapenko", "display_name_normalized": "Artem Astapenko", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "69713daee7ef", "image_original": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_original.jpg", "is_custom_image": true, "email": "jamakase54@gmail.com", "first_name": "Artem", "last_name": "Astapenko", "image_24": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_24.jpg", "image_32": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_32.jpg", "image_48": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_48.jpg", "image_72": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_72.jpg", "image_192": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_192.jpg", "image_512": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_512.jpg", "image_1024": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_1024.jpg", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1662549243, "is_email_confirmed": true, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397759} -{"stream": "channel_messages", "data": {"client_msg_id": "211e97d6-f529-4a61-adaf-65cbedd4c9d3", "type": "message", "text": " Good article from Temporal on use-case oriented docs structure", "user": "U01A6TEQXRU", "ts": "1627014630.175400", "team": "T01AB4DDR2N", "attachments": [{"title": "How we structured docs for user personas | Temporal documentation", "title_link": "https://docs.temporal.io/blog/docs-info-arch-2021/", "text": "A few weeks ago, around the end of March 2021, I set my focus on finally refreshing the information around Temporal's core concepts.", "fallback": "How we structured docs for user personas | Temporal documentation", "image_url": "https://docs.temporal.io/img/documentation-landing-page-april-2021.png", "from_url": "https://docs.temporal.io/blog/docs-info-arch-2021/", "image_width": 351, "image_height": 250, "image_bytes": 173793, "service_icon": "https://docs.temporal.io/img/favicon.png", "service_name": "docs.temporal.io", "id": 1, "original_url": "https://docs.temporal.io/blog/docs-info-arch-2021/"}], "blocks": [{"type": "rich_text", "block_id": "Il3Xn", "elements": [{"type": "rich_text_section", "elements": [{"type": "link", "url": "https://docs.temporal.io/blog/docs-info-arch-2021/"}, {"type": "text", "text": " Good article from Temporal on use-case oriented docs structure"}]}]}], "thread_ts": "1627014630.175400", "reply_count": 2, "reply_users_count": 2, "latest_reply": "1627331109.183100", "reply_users": ["U024VC21YRW", "U01MMSDJGC9"], "is_locked": false, "subscribed": false, "reactions": [{"name": "+1", "users": ["U027WFS9RV5", "U024VC21YRW", "U01MMSDJGC9", "U02QDDL2L74"], "count": 4}, {"name": "+1::skin-tone-4", "users": ["U01RW78A2LE"], "count": 1}], "channel_id": "C01AB7G87NE", "float_ts": 1627014630.1754}, "emitted_at": 1661589426355} -{"stream": "channel_messages", "data": {"client_msg_id": "65bf237c-29b0-452b-af93-af7f6c07bdce", "type": "message", "text": "Hi everyone,\nI have tried to replicate 10gb data from postgres as source to destination mysql but status still in progress ...it has been running since 2 hour.\ncan any one tell what will be reason ?", "user": "U026QUA5622", "ts": "1627062331.350600", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "=by4", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Hi everyone,\nI have tried to replicate 10gb data from postgres as source to destination mysql but status still in progress ...it has been running since 2 hour.\ncan any one tell what will be reason ?"}]}]}], "channel_id": "C021JANJ6TY", "float_ts": 1627062331.3506}, "emitted_at": 1661589426858} -{"stream": "channel_messages", "data": {"client_msg_id": "aaefcfbc-b33c-4435-aa14-99506353717e", "type": "message", "text": "Hello, I have set up airbyte for the first time with a Postgres > BigQuery connector, and I\u2019m facing a few issues and I\u2019m not sure whether I\u2019m doing it right, or if they are expected\n\u2022 I select only one table in the source schema, but once I saved I can\u2019t see anymore the \u201cnot selected\u201d ones, and when I update the source schema all tables get selected so I lose the information of which tables were previously selected\n\u2022 I get the following log at the end of the sync (though successful) : \n```2021-07-23 17:08:06 INFO () EnvConfigs(getEnvOrDefault):302 - WORKER_ENVIRONMENT not found or empty, defaulting to DOCKER\n2021-07-23 17:08:06 INFO () DefaultNormalizationWorker(run):77 - Normalization executed in 0.\n2021-07-23 17:08:06 INFO () TemporalAttemptExecution(get):133 - Stopping cancellation check scheduling...\n2021-07-23 17:08:06 ERROR () Exceptions(swallow):84 - Swallowed error.\njava.lang.NullPointerException: null value in entry: sync_cpu_request=null\n\tat com.google.common.collect.CollectPreconditions.checkEntryNotNull(CollectPreconditions.java:32) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap.entryOf(ImmutableMap.java:171) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap$Builder.put(ImmutableMap.java:281) ~[guava-30.1.1-jre.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.TrackingMetadata.generateSyncMetadata(TrackingMetadata.java:62) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.generateSyncMetadata(JobTracker.java:211) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.lambda$trackSync$3(JobTracker.java:134) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]```\nThanks !", "user": "U028HUR1TJR", "ts": "1627060754.349200", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "SS7FL", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Hello, I have set up airbyte for the first time with a Postgres > BigQuery connector, and I\u2019m facing a few issues and I\u2019m not sure whether I\u2019m doing it right, or if they are expected\n"}]}, {"type": "rich_text_list", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "I select only one table in the source schema, but once I saved I can\u2019t see anymore the \u201cnot selected\u201d ones, and when I update the source schema all tables get selected so I lose the information of which tables were previously selected"}]}, {"type": "rich_text_section", "elements": [{"type": "text", "text": "I get the following log at the end of the sync (though successful) : "}]}], "style": "bullet", "indent": 0}, {"type": "rich_text_preformatted", "elements": [{"type": "text", "text": "2021-07-23 17:08:06 INFO () EnvConfigs(getEnvOrDefault):302 - WORKER_ENVIRONMENT not found or empty, defaulting to DOCKER\n2021-07-23 17:08:06 INFO () DefaultNormalizationWorker(run):77 - Normalization executed in 0.\n2021-07-23 17:08:06 INFO () TemporalAttemptExecution(get):133 - Stopping cancellation check scheduling...\n2021-07-23 17:08:06 ERROR () Exceptions(swallow):84 - Swallowed error.\njava.lang.NullPointerException: null value in entry: sync_cpu_request=null\n\tat com.google.common.collect.CollectPreconditions.checkEntryNotNull(CollectPreconditions.java:32) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap.entryOf(ImmutableMap.java:171) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap$Builder.put(ImmutableMap.java:281) ~[guava-30.1.1-jre.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.TrackingMetadata.generateSyncMetadata(TrackingMetadata.java:62) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.generateSyncMetadata(JobTracker.java:211) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.lambda$trackSync$3(JobTracker.java:134) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]"}]}, {"type": "rich_text_section", "elements": [{"type": "text", "text": "Thanks !"}]}]}], "thread_ts": "1627060754.349200", "reply_count": 3, "reply_users_count": 2, "latest_reply": "1627062388.350800", "reply_users": ["U01BV1SDQMA", "U028HUR1TJR"], "is_locked": false, "subscribed": false, "channel_id": "C021JANJ6TY", "float_ts": 1627060754.3492}, "emitted_at": 1661589426858} -{"stream": "channel_messages", "data": {"client_msg_id": "3e57321f-da00-49dc-94b5-f3a0ef5aa723", "type": "message", "text": "Hey, we are currently gathering experience with our first connection (custom connector > redshift). We can\u2019t find much information on how the namespace customization works. Is it possible to inject own variables into the custom format as ${SOURCE_NAMESPACE}? Are there other variables that are offered by you in the default?", "user": "U027CJETYP8", "ts": "1627033140.340300", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "k=mGP", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Hey, we are currently gathering experience with our first connection (custom connector > redshift). We can\u2019t find much information on how the namespace customization works. Is it possible to inject own variables into the custom format as ${SOURCE_NAMESPACE}? Are there other variables that are offered by you in the default?"}]}]}], "thread_ts": "1627033140.340300", "reply_count": 5, "reply_users_count": 3, "latest_reply": "1627934001.030300", "reply_users": ["U01BV1SDQMA", "U027CJETYP8", "U01MMSDJGC9"], "is_locked": false, "subscribed": false, "channel_id": "C021JANJ6TY", "float_ts": 1627033140.3403}, "emitted_at": 1661589426858} -{"stream": "channel_messages", "data": {"client_msg_id": "092f4adb-d803-47ae-8c36-f6e2e99141b0", "type": "message", "text": "Hi all, I\u2019m testing airbyte deployment with GKE autopilot and it seems like HostPort/pod affinity is not permited in autopilot as the error below says.\n```kubectl apply -k kube/overlays/stable \n\nserviceaccount/airbyte-admin created\n created\n created\nconfigmap/airbyte-env-m2467g889g created\nconfigmap/airbyte-temporal-dynamicconfig created\nconfigmap/sweep-pod-script created\nsecret/gcs-log-creds created\nservice/airbyte-db-svc created\nservice/airbyte-minio-svc created\nservice/airbyte-server-svc created\nservice/airbyte-temporal-svc created\nservice/airbyte-webapp-svc created\ndeployment.apps/airbyte-db created\ndeployment.apps/airbyte-pod-sweeper created\ndeployment.apps/airbyte-scheduler created\ndeployment.apps/airbyte-temporal created\ndeployment.apps/airbyte-webapp created\npersistentvolumeclaim/airbyte-minio-pv-claim created\npersistentvolumeclaim/airbyte-volume-configs created\npersistentvolumeclaim/airbyte-volume-db created\npersistentvolumeclaim/airbyte-volume-workspace created\nError from server ([denied by autogke-no-host-port] container airbyte-minio specifies a host port; disallowed in Autopilot. error when creating \"kube/overlays/stable\": admission webhook \"validation.gatekeeper.sh\" denied the request: [denied by autogke-no-host-port] container airbyte-minio specifies a host port; disallowed in Autopilot. \nError from server ([denied by autogke-pod-affinity-limitation] pod affinity is not allowed in Autopilot on topologyKeys: <{\"kubernetes.io/hostname\"}>; Autopilot allows pod affinity only on topologyKeys: <[\"topology.kubernetes.io/region\", \"topology.kubernetes.io/zone\", \"failure-domain.beta.kubernetes.io/region\", \"failure-domain.beta.kubernetes.io/zone\"]>. error when creating \"kube/overlays/stable\": admission webhook \"validation.gatekeeper.sh\" denied the request: [denied by autogke-pod-affinity-limitation] pod affinity is not allowed in Autopilot on topologyKeys: <{\"kubernetes.io/hostname\"}>; Autopilot allows pod affinity only on topologyKeys: <[\"topology.kubernetes.io/region\", \"topology.kubernetes.io/zone\", \"failure-domain.beta.kubernetes.io/region\", \"failure-domain.beta.kubernetes.io/zone\"]>. ```\nAre there any workarounds on this issue?", "user": "U024WGT8H3R", "ts": "1627023042.338300", "team": "T01AB4DDR2N", "edited": {"user": "U024WGT8H3R", "ts": "1627023114.000000"}, "blocks": [{"type": "rich_text", "block_id": "4NDy", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Hi all, I\u2019m testing airbyte deployment with GKE autopilot and it seems like HostPort/pod affinity is not permited in autopilot as the error below says.\n"}]}, {"type": "rich_text_preformatted", "elements": [{"type": "text", "text": "kubectl apply -k kube/overlays/stable \n\nserviceaccount/airbyte-admin created\nrole.rbac.authorization.k8s.io/airbyte-admin-role created\nrolebinding.rbac.authorization.k8s.io/airbyte-admin-binding created\nconfigmap/airbyte-env-m2467g889g created\nconfigmap/airbyte-temporal-dynamicconfig created\nconfigmap/sweep-pod-script created\nsecret/gcs-log-creds created\nservice/airbyte-db-svc created\nservice/airbyte-minio-svc created\nservice/airbyte-server-svc created\nservice/airbyte-temporal-svc created\nservice/airbyte-webapp-svc created\ndeployment.apps/airbyte-db created\ndeployment.apps/airbyte-pod-sweeper created\ndeployment.apps/airbyte-scheduler created\ndeployment.apps/airbyte-temporal created\ndeployment.apps/airbyte-webapp created\npersistentvolumeclaim/airbyte-minio-pv-claim created\npersistentvolumeclaim/airbyte-volume-configs created\npersistentvolumeclaim/airbyte-volume-db created\npersistentvolumeclaim/airbyte-volume-workspace created\nError from server ([denied by autogke-no-host-port] container airbyte-minio specifies a host port; disallowed in Autopilot. error when creating \"kube/overlays/stable\": admission webhook \"validation.gatekeeper.sh\" denied the request: [denied by autogke-no-host-port] container airbyte-minio specifies a host port; disallowed in Autopilot. \nError from server ([denied by autogke-pod-affinity-limitation] pod affinity is not allowed in Autopilot on topologyKeys: <{\"kubernetes.io/hostname\"}>; Autopilot allows pod affinity only on topologyKeys: <[\"topology.kubernetes.io/region\", \"topology.kubernetes.io/zone\", \"failure-domain.beta.kubernetes.io/region\", \"failure-domain.beta.kubernetes.io/zone\"]>. error when creating \"kube/overlays/stable\": admission webhook \"validation.gatekeeper.sh\" denied the request: [denied by autogke-pod-affinity-limitation] pod affinity is not allowed in Autopilot on topologyKeys: <{\"kubernetes.io/hostname\"}>; Autopilot allows pod affinity only on topologyKeys: <[\"topology.kubernetes.io/region\", \"topology.kubernetes.io/zone\", \"failure-domain.beta.kubernetes.io/region\", \"failure-domain.beta.kubernetes.io/zone\"]>. "}]}, {"type": "rich_text_section", "elements": [{"type": "text", "text": "Are there any workarounds on this issue?"}]}]}], "thread_ts": "1627023042.338300", "reply_count": 9, "reply_users_count": 4, "latest_reply": "1632724785.457800", "reply_users": ["U01BV1SDQMA", "U01HYQYV1BQ", "U024WGT8H3R", "U02F7M6UD0S"], "is_locked": false, "subscribed": false, "reactions": [{"name": "white_check_mark", "users": ["U01HYQYV1BQ"], "count": 1}], "channel_id": "C021JANJ6TY", "float_ts": 1627023042.3383}, "emitted_at": 1661589426858} -{"stream": "threads", "data": {"client_msg_id": "211e97d6-f529-4a61-adaf-65cbedd4c9d3", "type": "message", "text": " Good article from Temporal on use-case oriented docs structure", "user": "U01A6TEQXRU", "ts": "1627014630.175400", "team": "T01AB4DDR2N", "attachments": [{"title": "How we structured docs for user personas | Temporal documentation", "title_link": "https://docs.temporal.io/blog/docs-info-arch-2021/", "text": "A few weeks ago, around the end of March 2021, I set my focus on finally refreshing the information around Temporal's core concepts.", "fallback": "How we structured docs for user personas | Temporal documentation", "image_url": "https://docs.temporal.io/img/documentation-landing-page-april-2021.png", "from_url": "https://docs.temporal.io/blog/docs-info-arch-2021/", "image_width": 351, "image_height": 250, "image_bytes": 173793, "service_icon": "https://docs.temporal.io/img/favicon.png", "service_name": "docs.temporal.io", "id": 1, "original_url": "https://docs.temporal.io/blog/docs-info-arch-2021/"}], "blocks": [{"type": "rich_text", "block_id": "Il3Xn", "elements": [{"type": "rich_text_section", "elements": [{"type": "link", "url": "https://docs.temporal.io/blog/docs-info-arch-2021/"}, {"type": "text", "text": " Good article from Temporal on use-case oriented docs structure"}]}]}], "thread_ts": "1627014630.175400", "reply_count": 2, "reply_users_count": 2, "latest_reply": "1627331109.183100", "reply_users": ["U024VC21YRW", "U01MMSDJGC9"], "is_locked": false, "subscribed": false, "reactions": [{"name": "+1", "users": ["U027WFS9RV5", "U024VC21YRW", "U01MMSDJGC9", "U02QDDL2L74"], "count": 4}, {"name": "+1::skin-tone-4", "users": ["U01RW78A2LE"], "count": 1}], "channel_id": "C01AB7G87NE", "float_ts": 1627014630.1754}, "emitted_at": 1661589428491} -{"stream": "threads", "data": {"client_msg_id": "6ae996d2-85c7-423f-b24e-9f6ce29c109e", "type": "message", "text": "Separating the ops/server persona from the dev persona is a major thing I look for in good vendor docs. I always had a few weeks of ops work to do with vendor adoption before I could hand stuff off to the dev team to actually use. I wanted the details about deployment nuances right up front, but not making a mess of the dev docs. So it\u2019s good to see that reflected in the article.", "user": "U024VC21YRW", "ts": "1627055946.177500", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "4xpJB", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Separating the ops/server persona from the dev persona is a major thing I look for in good vendor docs. I always had a few weeks of ops work to do with vendor adoption before I could hand stuff off to the dev team to actually use. I wanted the details about deployment nuances right up front, but not making a mess of the dev docs. So it\u2019s good to see that reflected in the article."}]}]}], "thread_ts": "1627014630.175400", "parent_user_id": "U01A6TEQXRU", "channel_id": "C01AB7G87NE", "float_ts": 1627055946.1775}, "emitted_at": 1661589428494} -{"stream": "threads", "data": {"client_msg_id": "00b54cbd-9fce-421d-9c4e-33a5bad08ab7", "type": "message", "text": "<@U01RW78A2LE> this is something that maybe we need to discuss :smile:", "user": "U01MMSDJGC9", "ts": "1627331109.183100", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "xBQ", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U01RW78A2LE"}, {"type": "text", "text": " this is something that maybe we need to discuss "}, {"type": "emoji", "name": "smile", "unicode": "1f604"}]}]}], "thread_ts": "1627014630.175400", "parent_user_id": "U01A6TEQXRU", "reactions": [{"name": "+1::skin-tone-4", "users": ["U01RW78A2LE"], "count": 1}], "channel_id": "C01AB7G87NE", "float_ts": 1627331109.1831}, "emitted_at": 1661589428494} -{"stream": "threads", "data": {"client_msg_id": "65bf237c-29b0-452b-af93-af7f6c07bdce", "type": "message", "text": "Hi everyone,\nI have tried to replicate 10gb data from postgres as source to destination mysql but status still in progress ...it has been running since 2 hour.\ncan any one tell what will be reason ?", "user": "U026QUA5622", "ts": "1627062331.350600", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "=by4", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Hi everyone,\nI have tried to replicate 10gb data from postgres as source to destination mysql but status still in progress ...it has been running since 2 hour.\ncan any one tell what will be reason ?"}]}]}], "channel_id": "C021JANJ6TY", "float_ts": 1627062331.3506}, "emitted_at": 1661589429206} -{"stream": "threads", "data": {"client_msg_id": "aaefcfbc-b33c-4435-aa14-99506353717e", "type": "message", "text": "Hello, I have set up airbyte for the first time with a Postgres > BigQuery connector, and I\u2019m facing a few issues and I\u2019m not sure whether I\u2019m doing it right, or if they are expected\n\u2022 I select only one table in the source schema, but once I saved I can\u2019t see anymore the \u201cnot selected\u201d ones, and when I update the source schema all tables get selected so I lose the information of which tables were previously selected\n\u2022 I get the following log at the end of the sync (though successful) : \n```2021-07-23 17:08:06 INFO () EnvConfigs(getEnvOrDefault):302 - WORKER_ENVIRONMENT not found or empty, defaulting to DOCKER\n2021-07-23 17:08:06 INFO () DefaultNormalizationWorker(run):77 - Normalization executed in 0.\n2021-07-23 17:08:06 INFO () TemporalAttemptExecution(get):133 - Stopping cancellation check scheduling...\n2021-07-23 17:08:06 ERROR () Exceptions(swallow):84 - Swallowed error.\njava.lang.NullPointerException: null value in entry: sync_cpu_request=null\n\tat com.google.common.collect.CollectPreconditions.checkEntryNotNull(CollectPreconditions.java:32) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap.entryOf(ImmutableMap.java:171) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap$Builder.put(ImmutableMap.java:281) ~[guava-30.1.1-jre.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.TrackingMetadata.generateSyncMetadata(TrackingMetadata.java:62) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.generateSyncMetadata(JobTracker.java:211) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.lambda$trackSync$3(JobTracker.java:134) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]```\nThanks !", "user": "U028HUR1TJR", "ts": "1627060754.349200", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "SS7FL", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Hello, I have set up airbyte for the first time with a Postgres > BigQuery connector, and I\u2019m facing a few issues and I\u2019m not sure whether I\u2019m doing it right, or if they are expected\n"}]}, {"type": "rich_text_list", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "I select only one table in the source schema, but once I saved I can\u2019t see anymore the \u201cnot selected\u201d ones, and when I update the source schema all tables get selected so I lose the information of which tables were previously selected"}]}, {"type": "rich_text_section", "elements": [{"type": "text", "text": "I get the following log at the end of the sync (though successful) : "}]}], "style": "bullet", "indent": 0}, {"type": "rich_text_preformatted", "elements": [{"type": "text", "text": "2021-07-23 17:08:06 INFO () EnvConfigs(getEnvOrDefault):302 - WORKER_ENVIRONMENT not found or empty, defaulting to DOCKER\n2021-07-23 17:08:06 INFO () DefaultNormalizationWorker(run):77 - Normalization executed in 0.\n2021-07-23 17:08:06 INFO () TemporalAttemptExecution(get):133 - Stopping cancellation check scheduling...\n2021-07-23 17:08:06 ERROR () Exceptions(swallow):84 - Swallowed error.\njava.lang.NullPointerException: null value in entry: sync_cpu_request=null\n\tat com.google.common.collect.CollectPreconditions.checkEntryNotNull(CollectPreconditions.java:32) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap.entryOf(ImmutableMap.java:171) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap$Builder.put(ImmutableMap.java:281) ~[guava-30.1.1-jre.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.TrackingMetadata.generateSyncMetadata(TrackingMetadata.java:62) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.generateSyncMetadata(JobTracker.java:211) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.lambda$trackSync$3(JobTracker.java:134) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]"}]}, {"type": "rich_text_section", "elements": [{"type": "text", "text": "Thanks !"}]}]}], "thread_ts": "1627060754.349200", "reply_count": 3, "reply_users_count": 2, "latest_reply": "1627062388.350800", "reply_users": ["U01BV1SDQMA", "U028HUR1TJR"], "is_locked": false, "subscribed": false, "channel_id": "C021JANJ6TY", "float_ts": 1627060754.3492}, "emitted_at": 1661589429515} -{"stream": "threads", "data": {"client_msg_id": "71a5a159-5e5c-462d-87ce-c202172dbb99", "type": "message", "text": "You can follow these issues, i think they are similar to what you are reporting:\n\u2022 \n\u2022 ", "user": "U01BV1SDQMA", "ts": "1627060931.349300", "team": "T01AB4DDR2N", "attachments": [{"title": "", "footer": "", "id": 1, "footer_icon": "https://slack.github.com/static/img/favicon-neutral.png", "ts": 1621556593, "color": "36a64f", "mrkdwn_in": ["text"], "fallback": "", "bot_id": "B01US8SR7JN", "app_unfurl_url": "https://github.com/airbytehq/airbyte/issues/3520", "is_app_unfurl": true}, {"title": "", "footer": "", "id": 2, "footer_icon": "https://slack.github.com/static/img/favicon-neutral.png", "ts": 1624460581, "color": "36a64f", "mrkdwn_in": ["text"], "fallback": "", "bot_id": "B01US8SR7JN", "app_unfurl_url": "https://github.com/airbytehq/airbyte/issues/4295", "is_app_unfurl": true}], "blocks": [{"type": "rich_text", "block_id": "5BgK", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "You can follow these issues, i think they are similar to what you are reporting:\n"}]}, {"type": "rich_text_list", "elements": [{"type": "rich_text_section", "elements": [{"type": "link", "url": "https://github.com/airbytehq/airbyte/issues/3520"}]}, {"type": "rich_text_section", "elements": [{"type": "link", "url": "https://github.com/airbytehq/airbyte/issues/4295"}]}], "style": "bullet", "indent": 0}]}], "thread_ts": "1627060754.349200", "parent_user_id": "U028HUR1TJR", "channel_id": "C021JANJ6TY", "float_ts": 1627060931.3493}, "emitted_at": 1661589429516} -{"stream": "threads", "data": {"client_msg_id": "bea4bda9-286a-494d-b9a0-e21799482588", "type": "message", "text": "For your exception, you don\u2019t have to worry about it but thanks for reporting! we\u2019ll have to address this but it\u2019s only for reporting usage of the app, there would be no impact on the sync job itself", "user": "U01BV1SDQMA", "ts": "1627061101.349600", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "a/t", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "For your exception, you don\u2019t have to worry about it but thanks for reporting! we\u2019ll have to address this but it\u2019s only for reporting usage of the app, there would be no impact on the sync job itself"}]}]}], "thread_ts": "1627060754.349200", "parent_user_id": "U028HUR1TJR", "reactions": [{"name": "+1", "users": ["U028HUR1TJR"], "count": 1}], "channel_id": "C021JANJ6TY", "float_ts": 1627061101.3496}, "emitted_at": 1661589429517} -{"stream": "threads", "data": {"client_msg_id": "ba278400-8e9a-4602-9a9e-8d19b4119d5b", "type": "message", "text": "Thanks !", "user": "U028HUR1TJR", "ts": "1627062388.350800", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "vXY6K", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Thanks !"}]}]}], "thread_ts": "1627060754.349200", "parent_user_id": "U028HUR1TJR", "channel_id": "C021JANJ6TY", "float_ts": 1627062388.3508}, "emitted_at": 1661589429518} -{"stream": "threads", "data": {"client_msg_id": "3e57321f-da00-49dc-94b5-f3a0ef5aa723", "type": "message", "text": "Hey, we are currently gathering experience with our first connection (custom connector > redshift). We can\u2019t find much information on how the namespace customization works. Is it possible to inject own variables into the custom format as ${SOURCE_NAMESPACE}? Are there other variables that are offered by you in the default?", "user": "U027CJETYP8", "ts": "1627033140.340300", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "k=mGP", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Hey, we are currently gathering experience with our first connection (custom connector > redshift). We can\u2019t find much information on how the namespace customization works. Is it possible to inject own variables into the custom format as ${SOURCE_NAMESPACE}? Are there other variables that are offered by you in the default?"}]}]}], "thread_ts": "1627033140.340300", "reply_count": 5, "reply_users_count": 3, "latest_reply": "1627934001.030300", "reply_users": ["U01BV1SDQMA", "U027CJETYP8", "U01MMSDJGC9"], "is_locked": false, "subscribed": false, "channel_id": "C021JANJ6TY", "float_ts": 1627033140.3403}, "emitted_at": 1661589429820} -{"stream": "threads", "data": {"client_msg_id": "98339578-a93b-42ba-b325-271dbe71bfb4", "type": "message", "text": "have you gone through: ?", "user": "U01BV1SDQMA", "ts": "1627033374.340400", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "F=5uE", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "have you gone through: "}, {"type": "link", "url": "https://docs.airbyte.io/understanding-airbyte/namespaces"}, {"type": "text", "text": " ?"}]}]}], "thread_ts": "1627033140.340300", "parent_user_id": "U027CJETYP8", "channel_id": "C021JANJ6TY", "float_ts": 1627033374.3404}, "emitted_at": 1661589429821} -{"stream": "channel_members", "data": {"member_id": "U01A6TEQXRU", "channel_id": "C01AB7G87NE"}, "emitted_at": 1661589431665} -{"stream": "channel_members", "data": {"member_id": "U01A81VHD45", "channel_id": "C01AB7G87NE"}, "emitted_at": 1661589431667} -{"stream": "channel_members", "data": {"member_id": "U01AB6V6NMQ", "channel_id": "C01AB7G87NE"}, "emitted_at": 1661589431668} -{"stream": "channel_members", "data": {"member_id": "U01ADKHBJKC", "channel_id": "C01AB7G87NE"}, "emitted_at": 1661589431668} -{"stream": "channel_members", "data": {"member_id": "U01ADSJ1D5H", "channel_id": "C01AB7G87NE"}, "emitted_at": 1661589431669} -{"stream": "channel_members", "data": {"member_id": "U01AS8LGX41", "channel_id": "C01AB7G87NE"}, "emitted_at": 1661589431669} -{"stream": "channel_members", "data": {"member_id": "U01AV3Y2MK9", "channel_id": "C01AB7G87NE"}, "emitted_at": 1661589431670} -{"stream": "channel_members", "data": {"member_id": "U01AW6ZB6KZ", "channel_id": "C01AB7G87NE"}, "emitted_at": 1661589431670} -{"stream": "channel_members", "data": {"member_id": "U01B0DNF207", "channel_id": "C01AB7G87NE"}, "emitted_at": 1661589431670} -{"stream": "channel_members", "data": {"member_id": "U01B22CCBF0", "channel_id": "C01AB7G87NE"}, "emitted_at": 1661589431671} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile b/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile index 67ad709b1674..0268dfbb1bd5 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile @@ -32,5 +32,5 @@ COPY source_tiktok_marketing ./source_tiktok_marketing ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.16 +LABEL io.airbyte.version=0.1.17 LABEL io.airbyte.name=airbyte/source-tiktok-marketing diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_basic.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_basic.json index 6c4951dc1bc5..1fe24b09642a 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_basic.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_basic.json @@ -8,9 +8,9 @@ "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "primary_key": [["id"]] }, - { "stream": { "name": "ads", @@ -18,10 +18,12 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["modify_time"], - "source_defined_primary_key": [["ads"]] + "source_defined_primary_key": [["ad_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["modify_time"], + "primary_key": [["ad_id"]] }, { "stream": { @@ -33,7 +35,9 @@ "source_defined_primary_key": [["adgroup_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["modify_time"], + "primary_key": [["adgroup_id"]] }, { "stream": { @@ -45,7 +49,9 @@ "source_defined_primary_key": [["campaign_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "primary_key": [["campaign_id"]], + "cursor_field": ["modify_time"] } ] } diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_reports_daily.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_reports_daily.json index 1514e91196b3..e39a974ba78e 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_reports_daily.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_reports_daily.json @@ -6,10 +6,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["ad_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["ad_id"]] }, { "stream": { @@ -17,10 +20,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["advertiser_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["advertiser_id"]] }, { "stream": { @@ -28,10 +34,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["adgroup_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["adgroup_id"]] }, { "stream": { @@ -39,10 +48,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["campaign_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["campaign_id"]] }, { @@ -51,10 +63,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["campaign_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["campaign_id"]] }, { "stream": { @@ -62,10 +77,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["adgroup_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["adgroup_id"]] }, { "stream": { @@ -73,10 +91,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["ad_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["ad_id"]] }, { "stream": { @@ -84,10 +105,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["advertiser_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["advertiser_id"]] } ] } diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_reports_lifetime.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_reports_lifetime.json index 6738190a078c..bbc7be439352 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_reports_lifetime.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_reports_lifetime.json @@ -6,10 +6,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["ad_id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["ad_id"]] }, { "stream": { @@ -17,10 +20,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["advertiser_id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["advertiser_id"]] }, { "stream": { @@ -28,10 +34,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["adgroup_id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["adgroup_id"]] }, { "stream": { @@ -39,10 +48,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["campaign_id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["campaign_id"]] }, { @@ -51,10 +63,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["advertiser_id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["advertiser_id"]] } ] } diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_with_day_granularity.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_with_day_granularity.json index 47dcd3c6c1f9..da31538ef919 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_with_day_granularity.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_with_day_granularity.json @@ -6,10 +6,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["ad_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["ad_id"]] }, { "stream": { @@ -17,10 +20,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["advertiser_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["advertiser_id"]] }, { "stream": { @@ -28,10 +34,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["adgroup_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["adgroup_id"]] }, { "stream": { @@ -39,10 +48,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["campaign_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["campaign_id"]] }, { @@ -51,10 +63,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["campaign_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["campaign_id"]] }, { "stream": { @@ -62,10 +77,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["adgroup_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["adgroup_id"]] }, { "stream": { @@ -73,10 +91,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["ad_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["ad_id"]] }, { "stream": { @@ -84,10 +105,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["advertiser_id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["advertiser_id"]] } ] } diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_with_lifetime_granularity.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_with_lifetime_granularity.json index f7c98e21909c..26a9bc141067 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_with_lifetime_granularity.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_with_lifetime_granularity.json @@ -6,10 +6,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["ad_id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["ad_id"]] }, { "stream": { @@ -17,10 +20,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["advertiser_id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["advertiser_id"]] }, { "stream": { @@ -28,10 +34,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["adgroup_id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["adgroup_id"]] }, { "stream": { @@ -39,10 +48,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["campaign_id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["campaign_id"]] }, { @@ -51,10 +63,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["stat_time_day"] + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [["advertiser_id"]] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["stat_time_day"], + "primary_key": [["advertiser_id"]] } ] } diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py index e10e0eef7779..855658f596f4 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py @@ -198,13 +198,14 @@ def should_retry(self, response: requests.Response) -> bool: """ Once the rate limit is met, the server returns "code": 40100 Docs: https://business-api.tiktok.com/marketing_api/docs?id=1701890997610497 + Retry 50002 as well - it's a server error. """ try: data = response.json() except Exception: self.logger.error(f"Incorrect JSON response: {response.text}") raise - if data["code"] == 40100: + if data["code"] in (40100, 50002): return True return super().should_retry(response) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py index ef7eb38767d7..17efde817bf8 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py @@ -35,13 +35,14 @@ def prepared_prod_args(): @timeout_decorator.timeout(20) -def test_backoff(prepared_sandbox_args): +@pytest.mark.parametrize("error_code", (40100, 50002)) +def test_backoff(prepared_sandbox_args, error_code): """TiktokMarketing sends the header 'Retry-After' about needed delay. All streams have to handle it""" stream = Advertisers(**prepared_sandbox_args) with requests_mock.Mocker() as m: url = stream.url_base + stream.path() - m.get(url, text=json.dumps({"code": 40100})) + m.get(url, text=json.dumps({"code": error_code})) with pytest.raises(UserDefinedBackoffException): list(stream.read_records(sync_mode=None)) diff --git a/docs/integrations/sources/tiktok-marketing.md b/docs/integrations/sources/tiktok-marketing.md index 3f64914edcfb..e64332e1e6b0 100644 --- a/docs/integrations/sources/tiktok-marketing.md +++ b/docs/integrations/sources/tiktok-marketing.md @@ -536,7 +536,8 @@ The connector is restricted by [requests limitation](https://ads.tiktok.com/mark ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------- | +|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------| +| 0.1.17 | 2022-10-04 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Retry error 50002 | | 0.1.16 | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream state | | 0.1.15 | 2022-08-30 | [16137](https://github.com/airbytehq/airbyte/pull/16137) | Fixed bug with normalization caused by unsupported nested cursor field | | 0.1.14 | 2022-06-29 | [13890](https://github.com/airbytehq/airbyte/pull/13890) | Removed granularity config option | From 57468ae1773cbf4060f9fcde1e1fcea389197545 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Tue, 4 Oct 2022 17:07:31 +0300 Subject: [PATCH 2/3] source tiktok marketing: upd changelog --- docs/integrations/sources/tiktok-marketing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/tiktok-marketing.md b/docs/integrations/sources/tiktok-marketing.md index e64332e1e6b0..1eae546492d6 100644 --- a/docs/integrations/sources/tiktok-marketing.md +++ b/docs/integrations/sources/tiktok-marketing.md @@ -537,7 +537,7 @@ The connector is restricted by [requests limitation](https://ads.tiktok.com/mark | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------| -| 0.1.17 | 2022-10-04 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Retry error 50002 | +| 0.1.17 | 2022-10-04 | [17557](https://github.com/airbytehq/airbyte/pull/17557) | Retry error 50002 | | 0.1.16 | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream state | | 0.1.15 | 2022-08-30 | [16137](https://github.com/airbytehq/airbyte/pull/16137) | Fixed bug with normalization caused by unsupported nested cursor field | | 0.1.14 | 2022-06-29 | [13890](https://github.com/airbytehq/airbyte/pull/13890) | Removed granularity config option | From 3f80b8eb61cc50133b0268f4215488954dfe799c Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Tue, 4 Oct 2022 18:04:35 +0000 Subject: [PATCH 3/3] auto-bump connector version [ci skip] --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index cdc38ccba530..6958683c1198 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1059,7 +1059,7 @@ - name: TikTok Marketing sourceDefinitionId: 4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35 dockerRepository: airbyte/source-tiktok-marketing - dockerImageTag: 0.1.16 + dockerImageTag: 0.1.17 documentationUrl: https://docs.airbyte.io/integrations/sources/tiktok-marketing icon: tiktok.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 696b7dad2361..6b3c88ad03b2 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -11181,7 +11181,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-tiktok-marketing:0.1.16" +- dockerImage: "airbyte/source-tiktok-marketing:0.1.17" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/tiktok-marketing" changelogUrl: "https://docs.airbyte.io/integrations/sources/tiktok-marketing"