diff --git a/airbyte-integrations/connectors/source-paypal-transaction/metadata.yaml b/airbyte-integrations/connectors/source-paypal-transaction/metadata.yaml index 3f8d1079e504..0e8df098b323 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/metadata.yaml +++ b/airbyte-integrations/connectors/source-paypal-transaction/metadata.yaml @@ -20,7 +20,7 @@ data: name: Paypal Transaction registries: cloud: - dockerImageTag: 2.0.0 #https://github.com/airbytehq/oncall/issues/3347 + dockerImageTag: 2.0.0 #https://github.com/airbytehq/oncall/issues/3347 enabled: true oss: enabled: true diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml b/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml index 484fd19e6468..767efd374086 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml @@ -50,6 +50,16 @@ acceptance_tests: ads: - name: "profile_image_url" bypass_reason: "Volatile URLs" + creative_assets_images: + - name: "image_url" + bypass_reason: "Volatile URL params" + creative_assets_videos: + - name: "preview_url" + bypass_reason: "Volatile URL params" + - name: "video_cover_url" + bypass_reason: "Volatile URL params" + - name: "preview_url_expire_time" + bypass_reason: "Changes over time" empty_streams: - name: ads_reports_hourly bypass_reason: "Tested with daily granularity." @@ -75,6 +85,10 @@ acceptance_tests: bypass_reason: "Stream is added by the Contributor. We should seed the sandbox later on." - name: ads_audience_reports_by_platform_daily bypass_reason: "Stream is added by the Contributor. We should seed the sandbox later on." + - name: creative_assets_portfolios + bypass_reason: "No data in the integration test account. We should seed the sandbox later on." + - name: creative_assets_music + bypass_reason: "System music provided by TikTok - very volatile data." # Old style streams with granularity config (for < 0.1.13) # Note: not needed to be tested separately in full and incremental tests, because code of # these streams is called directly in new style streams @@ -85,6 +99,20 @@ acceptance_tests: exact_order: no extra_records: yes timeout_seconds: 1200 + ignored_fields: + ads: + - name: "profile_image_url" + bypass_reason: "Volatile URLs" + creative_assets_images: + - name: "image_url" + bypass_reason: "Volatile URL params" + creative_assets_videos: + - name: "preview_url" + bypass_reason: "Volatile URL params" + - name: "video_cover_url" + bypass_reason: "Volatile URL params" + - name: "preview_url_expire_time" + bypass_reason: "Changes over time" empty_streams: - name: ad_groups bypass_reason: "This stream is tested on the new style config." @@ -96,6 +124,10 @@ acceptance_tests: bypass_reason: "This stream is tested on the new style config." - name: campaigns bypass_reason: "This stream is tested on the new style config." + - name: creative_assets_portfolios + bypass_reason: "No data in the integration test account. We should seed the sandbox later on." + - name: creative_assets_music + bypass_reason: "System music provided by TikTok - very volatile data." full_refresh: tests: - config_path: "secrets/config.json" @@ -105,6 +137,16 @@ acceptance_tests: ads: - name: "profile_image_url" bypass_reason: "Volatile URLs" + creative_assets_images: + - name: "image_url" + bypass_reason: "Volatile URL params" + creative_assets_videos: + - name: "preview_url" + bypass_reason: "Volatile URL params" + - name: "video_cover_url" + bypass_reason: "Volatile URL params" + - name: "preview_url_expire_time" + bypass_reason: "Changes over time" incremental: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json index 1ddb68e19891..485ecccb2315 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json @@ -119,5 +119,27 @@ "modify_time": "2099-01-01 01:00:0" } } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "creative_assets_images" + }, + "stream_state": { + "modify_time": "2099-01-01 01:00:0" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "creative_assets_videos" + }, + "stream_state": { + "modify_time": "2099-01-01 01:00:0" + } + } } ] diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/configured_catalog.json index 90acbeb472d2..b908a210eda8 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/configured_catalog.json @@ -183,6 +183,56 @@ "destination_sync_mode": "overwrite", "cursor_field": ["stat_time_day"], "primary_key": [["advertiser_id"], ["stat_time_day"], ["gender"], ["age"]] + }, + { + "stream": { + "name": "audiences", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["audience_id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "primary_key": [["audience_id"]] + }, + { + "stream": { + "name": "creative_assets_portfolios", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["creative_portfolio_id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "primary_key": [["creative_portfolio_id"]] + }, + { + "stream": { + "name": "creative_assets_images", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["modify_time"], + "source_defined_primary_key": [["image_id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "cursor_field": ["modify_time"], + "primary_key": [["image_id"]] + }, + { + "stream": { + "name": "creative_assets_videos", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["modify_time"], + "source_defined_primary_key": [["video_id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "cursor_field": ["modify_time"], + "primary_key": [["video_id"]] } ] } diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records.jsonl index 0ae92b74eeb5..55c81240fd99 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records.jsonl @@ -48,3 +48,11 @@ {"stream": "advertisers_audience_reports_lifetime", "data": {"dimensions": {"gender": "MALE", "age": "AGE_35_44", "advertiser_id": 7002238017842757633}, "metrics": {"ctr": "1.26", "cpc": "0.36", "spend": "31.56", "cpm": "4.58", "clicks": "87", "impressions": "6897"}, "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_35_44"}, "emitted_at": 1697639275347} {"stream": "advertisers_audience_reports_lifetime", "data": {"dimensions": {"gender": "FEMALE", "age": "AGE_55_100", "advertiser_id": 7002238017842757633}, "metrics": {"ctr": "0.00", "cpc": "0.00", "spend": "0.00", "cpm": "0.00", "clicks": "0", "impressions": "17"}, "advertiser_id": 7002238017842757633, "gender": "FEMALE", "age": "AGE_55_100"}, "emitted_at": 1697639275356} {"stream": "advertisers_audience_reports_lifetime", "data": {"dimensions": {"gender": "MALE", "age": "AGE_13_17", "advertiser_id": 7002238017842757633}, "metrics": {"ctr": "1.21", "cpc": "0.32", "spend": "7.13", "cpm": "3.92", "clicks": "22", "impressions": "1818"}, "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_13_17"}, "emitted_at": 1697639275364} +{"stream": "audiences", "data": {"is_valid": false, "audience_type": "Lead Generation", "expired_time": "2022-10-20 07:26:39", "create_time": "2021-10-20 07:26:39", "is_expiring": true, "cover_num": 0, "shared": false, "is_creator": true, "calculate_type": null, "audience_id": "125451003", "name": "Airbyte2"}, "emitted_at": 1697723597280} +{"stream": "audiences", "data": {"is_valid": false, "audience_type": "Website Audience", "expired_time": "2022-10-20 07:10:04", "create_time": "2021-10-20 07:10:04", "is_expiring": true, "cover_num": 0, "shared": false, "is_creator": true, "calculate_type": null, "audience_id": "125450951", "name": "Airbyte"}, "emitted_at": 1697723597284} +{"stream": "creative_assets_images", "data": {"material_id": "7080121373767221250", "signature": "2fb70f79f5b11f1bf102039bdd9315df", "image_url": "https://p19-ad-site-sign-sg.ibyteimg.com/v0201/7f371ff6f0764f8b8ef4f37d7b980d50~tplv-d5opwmad15-image.jpeg?x-expires=2013083646&x-signature=40Idxl%2BrjckMLC6YGb32p%2FK3v3g%3D", "is_carousel_usable": false, "size": 33342, "displayable": false, "image_id": "v0201/7f371ff6f0764f8b8ef4f37d7b980d50", "width": 720, "format": "jpeg", "create_time": "2022-03-28T12:11:34Z", "file_name": "7080121373767221250", "height": 1280, "modify_time": "2022-03-28T12:09:10Z"}, "emitted_at": 1697723646940} +{"stream": "creative_assets_images", "data": {"material_id": "7080116242086625281", "signature": "f564bd399410d9c271f79f0c785414e1", "image_url": "https://p21-ad-sg.ibyteimg.com/obj/ad-site-i18n-sg/10623349cd96274a8bf6650030e15214", "is_carousel_usable": true, "size": 117823, "displayable": false, "image_id": "ad-site-i18n-sg/10623349cd96274a8bf6650030e15214", "width": 720, "format": "jpeg", "create_time": "2022-03-28T11:50:49Z", "file_name": "a2_1648468243469.png", "height": 1280, "modify_time": "2022-03-28T11:50:50Z"}, "emitted_at": 1697723646943} +{"stream": "creative_assets_images", "data": {"material_id": "7080114557281419265", "signature": "68fa13980c042ab6f998bf771375ae15", "image_url": "https://p21-ad-sg.ibyteimg.com/obj/ad-site-i18n-sg/31e1ea531418f2526783eebea4d43ae3", "is_carousel_usable": true, "size": 216662, "displayable": false, "image_id": "ad-site-i18n-sg/31e1ea531418f2526783eebea4d43ae3", "width": 1280, "format": "jpeg", "create_time": "2022-03-28T11:42:28Z", "file_name": "air.png", "height": 720, "modify_time": "2022-03-28T11:42:29Z"}, "emitted_at": 1697723646945} +{"stream": "creative_assets_videos", "data": {"allowed_placements": ["PLACEMENT_TOPBUZZ", "PLACEMENT_TIKTOK", "PLACEMENT_HELO", "PLACEMENT_PANGLE", "PLACEMENT_GLOBAL_APP_BUNDLE"], "video_id": "v10033g50000ckmkpnbc77ucmin3t88g", "height": 1280, "video_cover_url": "http://p16-sign-sg.tiktokcdn.com/tos-alisg-p-0051c001-sg/oEnxmUxanQZbiaZzA9eAQfNdQBb1lzBIDRgVDL~tplv-noop.image?x-expires=1697745278&x-signature=AsbW5pdXMdXYFA3TJ3ZZ9hVFeR0%3D", "modify_time": "2023-10-16T14:46:55Z", "preview_url_expire_time": "2023-10-19 19:54:38", "format": "mp4", "displayable": true, "bit_rate": 3640400, "signature": "62a98cced4a71e2fe6e1edb77ca9dfb7", "material_id": "7290567877181833218", "preview_url": "http://v16m-default.akamaized.net/529e958f65b2c3924fcea48487ab8653/6531897e/video/tos/alisg/tos-alisg-ve-0051c001-sg/oUQJeNBlILamze9UZixnqggDQQBbAdaGxIGVDU/?a=0&ch=0&cr=0&dr=0&lr=ad&cd=0%7C0%7C0%7C0&cv=1&br=1666&bt=833&bti=Njs0Zi8tOg%3D%3D&cs=0&ds=3&ft=dl9~j-Inz7TqgOpZiyq8Z&mime_type=video_mp4&qs=0&rc=ZGc8ZmQ7Omg2Ozs7ZzhpZ0Bpajs2bGY6ZnFubjMzODYzNEAxYS8wLmMyNjQxMy0uNS8uYSM1cWtqcjRfbWpgLS1kMC1zcw%3D%3D&l=2023101913540719107ED36B1B69182C5C&btag=e00088000", "size": 13657417, "file_name": "Video16974675946002_Heartwarming Atmosphere Pops with Piano Main(827850)", "duration": 30.013, "create_time": "2023-10-16T14:46:55Z", "width": 720, "allow_download": true}, "emitted_at": 1697723649195} +{"stream": "creative_assets_videos", "data": {"allowed_placements": ["PLACEMENT_TOPBUZZ", "PLACEMENT_TIKTOK", "PLACEMENT_HELO", "PLACEMENT_PANGLE", "PLACEMENT_GLOBAL_APP_BUNDLE"], "video_id": "v10033g50000ckmkpmbc77u1rm8g4jg0", "height": 1280, "video_cover_url": "http://p16-sign-sg.tiktokcdn.com/tos-alisg-p-0051c001-sg/ogy2o5t3sATEqAfmBACIZNrAWlBEwWt8SZTzhg~tplv-noop.image?x-expires=1697745278&x-signature=U9UXLX%2FGpFVaXXDfmbYKJ%2FRw2mU%3D", "modify_time": "2023-10-16T14:46:54Z", "preview_url_expire_time": "2023-10-19 19:54:38", "format": "mp4", "displayable": true, "bit_rate": 3762666, "signature": "45aa9a03c4b7485e7b98251de9c1e48b", "material_id": "7290567861675278338", "preview_url": "http://v16m-default.akamaized.net/e3ddc851375bfc1da3eee62846f3e7ca/6531897e/video/tos/alisg/tos-alisg-ve-0051c001-sg/o4tZwlzQdEyGfICZsotAm3sOQBTlTqBEAhAgYI/?a=0&ch=0&cr=0&dr=0&lr=ad&cd=0%7C0%7C0%7C0&br=3524&bt=1762&bti=Njs0Zi8tOg%3D%3D&cs=0&ds=3&ft=dl9~j-Inz7TqgOpZiyq8Z&mime_type=video_mp4&qs=0&rc=aGY5PDxnO2U6ZDQ6NzYzaEBpM21qcDQ6ZnBubjMzODYzNEAvYzYtXjMtNjQxMmJgNGBfYSNkMTVvcjRfbWpgLS1kMC1zcw%3D%3D&l=2023101913540719107ED36B1B69182C5C&btag=e00088000", "size": 14116114, "file_name": "Video16974675945900_Whale, sea, electronica(859574)", "duration": 30.013, "create_time": "2023-10-16T14:46:54Z", "width": 720, "allow_download": true}, "emitted_at": 1697723649198} +{"stream": "creative_assets_videos", "data": {"allowed_placements": ["PLACEMENT_TOPBUZZ", "PLACEMENT_TIKTOK", "PLACEMENT_HELO", "PLACEMENT_PANGLE", "PLACEMENT_GLOBAL_APP_BUNDLE"], "video_id": "v10033g50000ckmkpm3c77u7v9shhf8g", "height": 1280, "video_cover_url": "http://p16-sign-sg.tiktokcdn.com/tos-alisg-p-0051c001-sg/ogelkgzkaAEPANUZgEeRLTQnYbnvYNUDBDB0tI~tplv-noop.image?x-expires=1697745278&x-signature=mdCZS0HP1tNoMIXWO6MSyKjlbgI%3D", "modify_time": "2023-10-16T14:46:50Z", "preview_url_expire_time": "2023-10-19 19:54:38", "format": "mp4", "displayable": true, "bit_rate": 3671087, "signature": "a492cfd6c89ba8312b159552b3a3c792", "material_id": "7290567856034070530", "preview_url": "http://v16m-default.akamaized.net/503be8b0ee76d4dec055e84343d31b0c/6531897e/video/tos/alisg/tos-alisg-ve-0051c001-sg/oUFNvZDEIBglY8LbnTeQtkoYRPzkQDAenBUNQP/?a=0&ch=0&cr=0&dr=0&lr=ad&cd=0%7C0%7C0%7C0&br=2846&bt=1423&bti=Njs0Zi8tOg%3D%3D&cs=0&ds=3&ft=dl9~j-Inz7TqgOpZiyq8Z&mime_type=video_mp4&qs=0&rc=aTMzNjZlaGdpOzZkZGlnN0BpamlrPDo6ZnBubjMzODYzNEA0XjBjYl5eNWAxM181M15jYSM1ZXBzcjQwbWpgLS1kMC1zcw%3D%3D&l=2023101913540719107ED36B1B69182C5C&btag=e00088000", "size": 13772545, "file_name": "Video16974675946053_U MAKE ME GO", "duration": 30.013, "create_time": "2023-10-16T14:46:50Z", "width": 720, "allow_download": true}, "emitted_at": 1697723649201} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records2.jsonl b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records2.jsonl index 70cb51a00652..5acd1660f7a7 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records2.jsonl +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records2.jsonl @@ -38,3 +38,11 @@ {"stream": "advertisers_audience_reports", "data": {"metrics": {"cpc": "0.29", "cpm": "3.88", "impressions": "2146", "ctr": "1.35", "clicks": "29", "spend": "8.32"}, "dimensions": {"stat_time_day": "2021-10-19 00:00:00", "age": "AGE_13_17", "advertiser_id": 7002238017842757633, "gender": "FEMALE"}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "gender": "FEMALE", "age": "AGE_13_17"}, "emitted_at": 1697524898170} {"stream": "advertisers_audience_reports", "data": {"metrics": {"cpc": "0.33", "cpm": "7.20", "impressions": "318", "ctr": "2.20", "clicks": "7", "spend": "2.29"}, "dimensions": {"stat_time_day": "2021-10-19 00:00:00", "age": "AGE_18_24", "advertiser_id": 7002238017842757633, "gender": "FEMALE"}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "gender": "FEMALE", "age": "AGE_18_24"}, "emitted_at": 1697524898171} {"stream": "advertisers_audience_reports", "data": {"metrics": {"cpc": "0.00", "cpm": "0.00", "impressions": "63", "ctr": "0.00", "clicks": "0", "spend": "0.00"}, "dimensions": {"stat_time_day": "2021-10-19 00:00:00", "age": "AGE_25_34", "advertiser_id": 7002238017842757633, "gender": "FEMALE"}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "gender": "FEMALE", "age": "AGE_25_34"}, "emitted_at": 1697524898173} +{"stream": "audiences", "data": {"is_valid": false, "audience_type": "Lead Generation", "expired_time": "2022-10-20 07:26:39", "create_time": "2021-10-20 07:26:39", "is_expiring": true, "cover_num": 0, "shared": false, "is_creator": true, "calculate_type": null, "audience_id": "125451003", "name": "Airbyte2"}, "emitted_at": 1697723597280} +{"stream": "audiences", "data": {"is_valid": false, "audience_type": "Website Audience", "expired_time": "2022-10-20 07:10:04", "create_time": "2021-10-20 07:10:04", "is_expiring": true, "cover_num": 0, "shared": false, "is_creator": true, "calculate_type": null, "audience_id": "125450951", "name": "Airbyte"}, "emitted_at": 1697723597284} +{"stream": "creative_assets_images", "data": {"material_id": "7080121373767221250", "signature": "2fb70f79f5b11f1bf102039bdd9315df", "image_url": "https://p19-ad-site-sign-sg.ibyteimg.com/v0201/7f371ff6f0764f8b8ef4f37d7b980d50~tplv-d5opwmad15-image.jpeg?x-expires=2013083646&x-signature=40Idxl%2BrjckMLC6YGb32p%2FK3v3g%3D", "is_carousel_usable": false, "size": 33342, "displayable": false, "image_id": "v0201/7f371ff6f0764f8b8ef4f37d7b980d50", "width": 720, "format": "jpeg", "create_time": "2022-03-28T12:11:34Z", "file_name": "7080121373767221250", "height": 1280, "modify_time": "2022-03-28T12:09:10Z"}, "emitted_at": 1697723646940} +{"stream": "creative_assets_images", "data": {"material_id": "7080116242086625281", "signature": "f564bd399410d9c271f79f0c785414e1", "image_url": "https://p21-ad-sg.ibyteimg.com/obj/ad-site-i18n-sg/10623349cd96274a8bf6650030e15214", "is_carousel_usable": true, "size": 117823, "displayable": false, "image_id": "ad-site-i18n-sg/10623349cd96274a8bf6650030e15214", "width": 720, "format": "jpeg", "create_time": "2022-03-28T11:50:49Z", "file_name": "a2_1648468243469.png", "height": 1280, "modify_time": "2022-03-28T11:50:50Z"}, "emitted_at": 1697723646943} +{"stream": "creative_assets_images", "data": {"material_id": "7080114557281419265", "signature": "68fa13980c042ab6f998bf771375ae15", "image_url": "https://p21-ad-sg.ibyteimg.com/obj/ad-site-i18n-sg/31e1ea531418f2526783eebea4d43ae3", "is_carousel_usable": true, "size": 216662, "displayable": false, "image_id": "ad-site-i18n-sg/31e1ea531418f2526783eebea4d43ae3", "width": 1280, "format": "jpeg", "create_time": "2022-03-28T11:42:28Z", "file_name": "air.png", "height": 720, "modify_time": "2022-03-28T11:42:29Z"}, "emitted_at": 1697723646945} +{"stream": "creative_assets_videos", "data": {"allowed_placements": ["PLACEMENT_TOPBUZZ", "PLACEMENT_TIKTOK", "PLACEMENT_HELO", "PLACEMENT_PANGLE", "PLACEMENT_GLOBAL_APP_BUNDLE"], "video_id": "v10033g50000ckmkpnbc77ucmin3t88g", "height": 1280, "video_cover_url": "http://p16-sign-sg.tiktokcdn.com/tos-alisg-p-0051c001-sg/oEnxmUxanQZbiaZzA9eAQfNdQBb1lzBIDRgVDL~tplv-noop.image?x-expires=1697745278&x-signature=AsbW5pdXMdXYFA3TJ3ZZ9hVFeR0%3D", "modify_time": "2023-10-16T14:46:55Z", "preview_url_expire_time": "2023-10-19 19:54:38", "format": "mp4", "displayable": true, "bit_rate": 3640400, "signature": "62a98cced4a71e2fe6e1edb77ca9dfb7", "material_id": "7290567877181833218", "preview_url": "http://v16m-default.akamaized.net/529e958f65b2c3924fcea48487ab8653/6531897e/video/tos/alisg/tos-alisg-ve-0051c001-sg/oUQJeNBlILamze9UZixnqggDQQBbAdaGxIGVDU/?a=0&ch=0&cr=0&dr=0&lr=ad&cd=0%7C0%7C0%7C0&cv=1&br=1666&bt=833&bti=Njs0Zi8tOg%3D%3D&cs=0&ds=3&ft=dl9~j-Inz7TqgOpZiyq8Z&mime_type=video_mp4&qs=0&rc=ZGc8ZmQ7Omg2Ozs7ZzhpZ0Bpajs2bGY6ZnFubjMzODYzNEAxYS8wLmMyNjQxMy0uNS8uYSM1cWtqcjRfbWpgLS1kMC1zcw%3D%3D&l=2023101913540719107ED36B1B69182C5C&btag=e00088000", "size": 13657417, "file_name": "Video16974675946002_Heartwarming Atmosphere Pops with Piano Main(827850)", "duration": 30.013, "create_time": "2023-10-16T14:46:55Z", "width": 720, "allow_download": true}, "emitted_at": 1697723649195} +{"stream": "creative_assets_videos", "data": {"allowed_placements": ["PLACEMENT_TOPBUZZ", "PLACEMENT_TIKTOK", "PLACEMENT_HELO", "PLACEMENT_PANGLE", "PLACEMENT_GLOBAL_APP_BUNDLE"], "video_id": "v10033g50000ckmkpmbc77u1rm8g4jg0", "height": 1280, "video_cover_url": "http://p16-sign-sg.tiktokcdn.com/tos-alisg-p-0051c001-sg/ogy2o5t3sATEqAfmBACIZNrAWlBEwWt8SZTzhg~tplv-noop.image?x-expires=1697745278&x-signature=U9UXLX%2FGpFVaXXDfmbYKJ%2FRw2mU%3D", "modify_time": "2023-10-16T14:46:54Z", "preview_url_expire_time": "2023-10-19 19:54:38", "format": "mp4", "displayable": true, "bit_rate": 3762666, "signature": "45aa9a03c4b7485e7b98251de9c1e48b", "material_id": "7290567861675278338", "preview_url": "http://v16m-default.akamaized.net/e3ddc851375bfc1da3eee62846f3e7ca/6531897e/video/tos/alisg/tos-alisg-ve-0051c001-sg/o4tZwlzQdEyGfICZsotAm3sOQBTlTqBEAhAgYI/?a=0&ch=0&cr=0&dr=0&lr=ad&cd=0%7C0%7C0%7C0&br=3524&bt=1762&bti=Njs0Zi8tOg%3D%3D&cs=0&ds=3&ft=dl9~j-Inz7TqgOpZiyq8Z&mime_type=video_mp4&qs=0&rc=aGY5PDxnO2U6ZDQ6NzYzaEBpM21qcDQ6ZnBubjMzODYzNEAvYzYtXjMtNjQxMmJgNGBfYSNkMTVvcjRfbWpgLS1kMC1zcw%3D%3D&l=2023101913540719107ED36B1B69182C5C&btag=e00088000", "size": 14116114, "file_name": "Video16974675945900_Whale, sea, electronica(859574)", "duration": 30.013, "create_time": "2023-10-16T14:46:54Z", "width": 720, "allow_download": true}, "emitted_at": 1697723649198} +{"stream": "creative_assets_videos", "data": {"allowed_placements": ["PLACEMENT_TOPBUZZ", "PLACEMENT_TIKTOK", "PLACEMENT_HELO", "PLACEMENT_PANGLE", "PLACEMENT_GLOBAL_APP_BUNDLE"], "video_id": "v10033g50000ckmkpm3c77u7v9shhf8g", "height": 1280, "video_cover_url": "http://p16-sign-sg.tiktokcdn.com/tos-alisg-p-0051c001-sg/ogelkgzkaAEPANUZgEeRLTQnYbnvYNUDBDB0tI~tplv-noop.image?x-expires=1697745278&x-signature=mdCZS0HP1tNoMIXWO6MSyKjlbgI%3D", "modify_time": "2023-10-16T14:46:50Z", "preview_url_expire_time": "2023-10-19 19:54:38", "format": "mp4", "displayable": true, "bit_rate": 3671087, "signature": "a492cfd6c89ba8312b159552b3a3c792", "material_id": "7290567856034070530", "preview_url": "http://v16m-default.akamaized.net/503be8b0ee76d4dec055e84343d31b0c/6531897e/video/tos/alisg/tos-alisg-ve-0051c001-sg/oUFNvZDEIBglY8LbnTeQtkoYRPzkQDAenBUNQP/?a=0&ch=0&cr=0&dr=0&lr=ad&cd=0%7C0%7C0%7C0&br=2846&bt=1423&bti=Njs0Zi8tOg%3D%3D&cs=0&ds=3&ft=dl9~j-Inz7TqgOpZiyq8Z&mime_type=video_mp4&qs=0&rc=aTMzNjZlaGdpOzZkZGlnN0BpamlrPDo6ZnBubjMzODYzNEA0XjBjYl5eNWAxM181M15jYSM1ZXBzcjQwbWpgLS1kMC1zcw%3D%3D&l=2023101913540719107ED36B1B69182C5C&btag=e00088000", "size": 13772545, "file_name": "Video16974675946053_U MAKE ME GO", "duration": 30.013, "create_time": "2023-10-16T14:46:50Z", "width": 720, "allow_download": true}, "emitted_at": 1697723649201} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml b/airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml index 77ca3935d55e..c690bf3a1c3c 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml @@ -11,7 +11,7 @@ data: connectorSubtype: api connectorType: source definitionId: 4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35 - dockerImageTag: 3.7.1 + dockerImageTag: 3.8.0 dockerRepository: airbyte/source-tiktok-marketing documentationUrl: https://docs.airbyte.com/integrations/sources/tiktok-marketing githubIssueLabel: source-tiktok-marketing diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/audiences.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/audiences.json new file mode 100644 index 000000000000..3559ce877e3a --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/audiences.json @@ -0,0 +1,40 @@ +{ + "type": "object", + "properties": { + "shared": { + "type": ["null", "boolean"] + }, + "is_creator": { + "type": ["null", "boolean"] + }, + "audience_id": { + "type": ["null", "string"] + }, + "cover_num": { + "type": ["null", "integer"] + }, + "create_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "is_valid": { + "type": ["null", "boolean"] + }, + "is_expiring": { + "type": ["null", "boolean"] + }, + "expired_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "name": { + "type": ["null", "string"] + }, + "audience_type": { + "type": ["null", "string"] + }, + "calculate_type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_images.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_images.json new file mode 100644 index 000000000000..5bee3b1723ca --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_images.json @@ -0,0 +1,46 @@ +{ + "type": "object", + "properties": { + "image_id": { + "type": ["null", "string"] + }, + "format": { + "type": ["null", "string"] + }, + "image_url": { + "type": ["null", "string"] + }, + "height": { + "type": ["null", "integer"] + }, + "width": { + "type": ["null", "integer"] + }, + "signature": { + "type": ["null", "string"] + }, + "size": { + "type": ["null", "integer"] + }, + "material_id": { + "type": ["null", "string"] + }, + "is_carousel_usable": { + "type": ["null", "boolean"] + }, + "file_name": { + "type": ["null", "string"] + }, + "create_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "modify_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "displayable": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_music.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_music.json new file mode 100644 index 000000000000..06bb4ff374e2 --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_music.json @@ -0,0 +1,55 @@ +{ + "type": "object", + "properties": { + "music_id": { + "type": ["null", "string"] + }, + "material_id": { + "type": ["null", "string"] + }, + "sources": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "author": { + "type": ["null", "string"] + }, + "liked": { + "type": ["null", "boolean"] + }, + "cover_url": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "duration": { + "type": ["null", "number"] + }, + "style": { + "type": ["null", "string"] + }, + "signature": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "file_name": { + "type": ["null", "string"] + }, + "copyright": { + "type": ["null", "string"] + }, + "create_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "modify_time": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_portfolios.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_portfolios.json new file mode 100644 index 000000000000..2e9c74de2e8b --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_portfolios.json @@ -0,0 +1,14 @@ +{ + "type": "object", + "properties": { + "creative_portfolio_id": { + "type": ["null", "string"] + }, + "creative_portfolio_type": { + "type": ["null", "string"] + }, + "creative_portfolio_preview_url": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_videos.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_videos.json new file mode 100644 index 000000000000..72a3784310ab --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_videos.json @@ -0,0 +1,64 @@ +{ + "type": "object", + "properties": { + "video_id": { + "type": ["null", "string"] + }, + "video_cover_url": { + "type": ["null", "string"] + }, + "format": { + "type": ["null", "string"] + }, + "preview_url": { + "type": ["null", "string"] + }, + "preview_url_expire_time": { + "type": ["null", "string"] + }, + "duration": { + "type": ["null", "number"] + }, + "height": { + "type": ["null", "integer"] + }, + "width": { + "type": ["null", "integer"] + }, + "bit_rate": { + "type": ["null", "number"] + }, + "signature": { + "type": ["null", "string"] + }, + "size": { + "type": ["null", "integer"] + }, + "material_id": { + "type": ["null", "string"] + }, + "allowed_placements": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "allow_download": { + "type": ["null", "boolean"] + }, + "file_name": { + "type": ["null", "string"] + }, + "create_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "modify_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "displayable": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py index fa56f36cb626..2ef76e6db842 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py @@ -31,12 +31,17 @@ AdvertisersAudienceReportsByCountry, AdvertisersAudienceReportsByPlatform, AdvertisersReports, + Audiences, BasicReports, Campaigns, CampaignsAudienceReports, CampaignsAudienceReportsByCountry, CampaignsAudienceReportsByPlatform, CampaignsReports, + CreativeAssetsImages, + CreativeAssetsMusic, + CreativeAssetsPortfolios, + CreativeAssetsVideos, Daily, Hourly, Lifetime, @@ -134,7 +139,12 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Advertisers(**args), Ads(**args), AdGroups(**args), + Audiences(**args), Campaigns(**args), + CreativeAssetsImages(**args), + CreativeAssetsMusic(**args), + CreativeAssetsPortfolios(**args), + CreativeAssetsVideos(**args), ] if is_production: diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py index dfb4eb33bf75..243784b2d6c8 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py @@ -323,23 +323,6 @@ def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: def is_finished(self): return len(self._advertiser_ids) == 0 - def request_params( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - params = {"page_size": self.page_size} - if self.fields: - params["fields"] = self.convert_array_param(self.fields) - if stream_slice: - params.update(stream_slice) - return params - - -class IncrementalTiktokStream(FullRefreshTiktokStream, ABC): - cursor_field = "modify_time" - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """All responses have the following pagination data: { @@ -355,17 +338,32 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, } """ - page_info = response.json()["data"]["page_info"] + page_info = response.json().get("data", {}).get("page_info", {}) + if not page_info: + return None if page_info["page"] < page_info["total_page"]: return {"page": page_info["page"] + 1} return None - def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: - params = super().request_params(next_page_token=next_page_token, **kwargs) + def request_params( + self, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + params = {"page_size": self.page_size} + if self.fields: + params["fields"] = self.convert_array_param(self.fields) + if stream_slice: + params.update(stream_slice) if next_page_token: params.update(next_page_token) return params + +class IncrementalTiktokStream(FullRefreshTiktokStream, ABC): + cursor_field = "modify_time" + def select_cursor_field_value(self, data: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None) -> str: if not data or not self.cursor_field: return None @@ -457,6 +455,37 @@ def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: yield {"advertiser_ids": ids[i : min(end, i + step)]} +class Audiences(FullRefreshTiktokStream): + """Docs: https://business-api.tiktok.com/portal/docs?id=1739940506015746""" + + page_size = 100 + primary_key = "audience_id" + + def path(self, *args, **kwargs) -> str: + return "dmp/custom_audience/list/" + + +class CreativeAssetsMusic(FullRefreshTiktokStream): + """Docs: https://business-api.tiktok.com/portal/docs?id=1740053909509122""" + + primary_key = "music_id" + response_list_field = "musics" + + def path(self, *args, **kwargs) -> str: + return "file/music/get/" + + +class CreativeAssetsPortfolios(FullRefreshTiktokStream): + """Docs: https://business-api.tiktok.com/portal/docs?id=1766324010279938""" + + page_size = 100 + primary_key = "creative_portfolio_id" + response_list_field = "creative_portfolios" + + def path(self, *args, **kwargs) -> str: + return "creative/portfolio/list/" + + class Campaigns(IncrementalTiktokStream): """Docs: https://ads.tiktok.com/marketing_api/docs?id=1739315828649986""" @@ -484,6 +513,26 @@ def path(self, *args, **kwargs) -> str: return "ad/get/" +class CreativeAssetsImages(IncrementalTiktokStream): + """Docs: https://business-api.tiktok.com/portal/docs?id=1740052016789506""" + + page_size = 100 + primary_key = "image_id" + + def path(self, *args, **kwargs) -> str: + return "file/image/ad/search/" + + +class CreativeAssetsVideos(IncrementalTiktokStream): + """Docs: https://business-api.tiktok.com/portal/docs?id=1740050472224769""" + + page_size = 100 + primary_key = "video_id" + + def path(self, *args, **kwargs) -> str: + return "file/video/ad/search/" + + class BasicReports(IncrementalTiktokStream, ABC): """Docs: https://ads.tiktok.com/marketing_api/docs?id=1738864915188737""" diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py index d4ba815b782d..a2c4975260d2 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py @@ -9,3 +9,8 @@ @pytest.fixture(autouse=True) def patch_sleep(mocker): mocker.patch("time.sleep") + + +@pytest.fixture(autouse=True) +def disable_cache(mocker): + mocker.patch("source_tiktok_marketing.streams.AdvertiserIds.use_cache", new_callable=mocker.PropertyMock, return_value=False) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py index 1ace5637eb96..72a3838f01fe 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py @@ -134,8 +134,8 @@ def test_random_items(prepared_prod_args): @pytest.mark.parametrize( "config, stream_len", [ - (PROD_CONFIG_FILE, 30), - (SANDBOX_CONFIG_FILE, 22), + (PROD_CONFIG_FILE, 35), + (SANDBOX_CONFIG_FILE, 27), ], ) def test_source_streams(config, stream_len): diff --git a/docs/integrations/sources/tiktok-marketing.md b/docs/integrations/sources/tiktok-marketing.md index ac2cd9c8c01d..f2b6fbee9d85 100644 --- a/docs/integrations/sources/tiktok-marketing.md +++ b/docs/integrations/sources/tiktok-marketing.md @@ -64,7 +64,7 @@ To access the Sandbox environment: ## Supported streams and sync modes | Stream | Environment | Key | Incremental | -| :---------------------------------------- | ------------ | ------------------------------------------ | :---------- | +|:------------------------------------------| ------------ |--------------------------------------------|:------------| | Advertisers | Prod,Sandbox | advertiser_id | No | | AdGroups | Prod,Sandbox | adgroup_id | Yes | | Ads | Prod,Sandbox | ad_id | Yes | @@ -78,9 +78,14 @@ To access the Sandbox environment: | AdGroupsReportsHourly | Prod,Sandbox | adgroup_id, stat_time_hour | Yes | | AdGroupsReportsDaily | Prod,Sandbox | adgroup_id, stat_time_day | Yes | | AdGroupsReportsLifetime | Prod,Sandbox | adgroup_id | No | +| Audiences | Prod,Sandbox | audience_id | No | | CampaignsReportsHourly | Prod,Sandbox | campaign_id, stat_time_hour | Yes | | CampaignsReportsDaily | Prod,Sandbox | campaign_id, stat_time_day | Yes | | CampaignsReportsLifetime | Prod,Sandbox | campaign_id | No | +| CreativeAssetsImages | Prod,Sandbox | image_id | Yes | +| CreativeAssetsMusic | Prod,Sandbox | music_id | Yes | +| CreativeAssetsPortfolio | Prod,Sandbox | creative_portfolio_id | No | +| CreativeAssetsVideo | Prod,Sandbox | video_id | Yes | | AdvertisersAudienceReportsDaily | Prod | advertiser_id, stat_time_day, gender, age | Yes | | AdvertisersAudienceReportsByCountryDaily | Prod | advertiser_id, stat_time_day, country_code | Yes | | AdvertisersAudienceReportsByPlatformDaily | Prod | advertiser_id, stat_time_day, platform | Yes | @@ -581,7 +586,8 @@ The connector is restricted by [requests limitation](https://ads.tiktok.com/mark | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------| -| 3.7.1 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 3.8.0 | 2023-10-19 | [31610](https://github.com/airbytehq/airbyte/pull/31610) | Add Creative Assets and Audiences streams | +| 3.7.1 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | | 3.7.0 | 2023-10-19 | [31493](https://github.com/airbytehq/airbyte/pull/31493) | Add fields to Ads stream | | 3.6.0 | 2023-10-18 | [31537](https://github.com/airbytehq/airbyte/pull/31537) | Use default availability strategy | | 3.5.0 | 2023-10-16 | [31445](https://github.com/airbytehq/airbyte/pull/31445) | Apply minimum date restrictions |