From 454f84d3ed61081d7a915e7f60b1133a71654476 Mon Sep 17 00:00:00 2001 From: Ganpat Agarwal Date: Mon, 3 Oct 2022 19:54:33 +0530 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20Source=20Amazon=20Ads=20:=20Add?= =?UTF-8?q?=20attribution=20reports=20(#16342)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * use data field for json response * add attribution reports * update changelog * add atrribution report integration test * clean up expected_records add empty streams to acceptance test config * handle exception for profile * update tests Co-authored-by: Yiyang Li Co-authored-by: Harshith Mullapudi Co-authored-by: Sajarin --- .../acceptance-test-config.yml | 19 +- .../integration_tests/configured_catalog.json | 36 ++++ .../integration_tests/expected_records.txt | 122 ------------ .../source_amazon_ads/schemas/__init__.py | 2 + .../schemas/attribution_report.py | 30 +++ .../source_amazon_ads/source.py | 8 + .../source_amazon_ads/streams/__init__.py | 10 + .../streams/attribution_report.py | 175 ++++++++++++++++++ .../source_amazon_ads/streams/common.py | 7 +- .../source-amazon-ads/unit_tests/conftest.py | 119 ++++++++++++ .../unit_tests/test_attribution_report.py | 148 +++++++++++++++ .../unit_tests/test_source.py | 6 +- docs/integrations/sources/amazon-ads.md | 4 +- 13 files changed, 555 insertions(+), 131 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records.txt create mode 100644 airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/attribution_report.py create mode 100644 airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/attribution_report.py create mode 100644 airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_attribution_report.py diff --git a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml index b87df3eea52c..5d05abea8368 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml @@ -14,12 +14,19 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ["sponsored_brands_campaigns", "sponsored_brands_ad_groups", "sponsored_brands_keywords"] - expect_records: - path: "integration_tests/expected_records.txt" - extra_fields: no - exact_order: no - extra_records: no + empty_streams: + [ + "sponsored_brands_ad_groups", + "sponsored_brands_campaigns", + "sponsored_brands_keywords", + "sponsored_product_keywords", + "sponsored_product_negative_keywords", + "sponsored_product_targetings", + "attribution_report_performance_creative", + "attribution_report_performance_adgroup", + "attribution_report_products", + "attribution_report_performance_campaign", + ] - config_path: "secrets/config_report.json" configured_catalog_path: "integration_tests/configured_catalog_report.json" timeout_seconds: 2400 diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog.json index 31e97a02276f..8394bbf8feff 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog.json @@ -139,6 +139,42 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "attribution_report_products", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "attribution_report_performance_adgroup", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "attribution_report_performance_campaign", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "attribution_report_performance_creative", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records.txt deleted file mode 100644 index d74c774f8342..000000000000 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records.txt +++ /dev/null @@ -1,122 +0,0 @@ -{"stream":"profiles","data":{"profileId":3991703629696934,"countryCode":"CA","currencyCode":"CAD","dailyBudget":999999999,"timezone":"America/Los_Angeles","accountInfo":{"marketplaceStringId":"A2EUQ1WTGCTBG2","id":"A3LUQZ2NBMFGO4","type":"seller","name":"The Airbyte Store","validPaymentMethod":true}},"emitted_at":1659020216524} -{"stream":"profiles","data":{"profileId":2935840597082037,"countryCode":"CA","currencyCode":"CAD","timezone":"America/Los_Angeles","accountInfo":{"marketplaceStringId":"A2EUQ1WTGCTBG2","id":"ENTITY1T4PQ8E0Y1LVJ","type":"vendor","name":"test","validPaymentMethod":false}},"emitted_at":1659020216526} -{"stream":"profiles","data":{"profileId":3664951271230581,"countryCode":"MX","currencyCode":"MXN","dailyBudget":999999999,"timezone":"America/Los_Angeles","accountInfo":{"marketplaceStringId":"A1AM78C64UM0Y8","id":"A3LUQZ2NBMFGO4","type":"seller","name":"The Airbyte Store","validPaymentMethod":true}},"emitted_at":1659020216527} -{"stream":"profiles","data":{"profileId":1861552880916640,"countryCode":"US","currencyCode":"USD","timezone":"America/Los_Angeles","accountInfo":{"marketplaceStringId":"ATVPDKIKX0DER","id":"ENTITYVFIQ1E6W9INI","type":"vendor","name":"Sponsored ads - KDP","subType":"KDP_AUTHOR","validPaymentMethod":true}},"emitted_at":1659020216527} -{"stream":"profiles","data":{"profileId":3312910465837761,"countryCode":"US","currencyCode":"USD","dailyBudget":999999999,"timezone":"America/Los_Angeles","accountInfo":{"marketplaceStringId":"ATVPDKIKX0DER","id":"A3LUQZ2NBMFGO4","type":"seller","name":"The Airbyte Store","validPaymentMethod":true}},"emitted_at":1659020216527} -{"stream":"profiles","data":{"profileId":2445745172318948,"countryCode":"US","currencyCode":"USD","timezone":"America/Los_Angeles","accountInfo":{"marketplaceStringId":"ATVPDKIKX0DER","id":"ENTITY3QRPN1GHC1Q0U","type":"vendor","name":"3PTestBrand-A3LUQZ2NBMFGO46750119612846","validPaymentMethod":true}},"emitted_at":1659020216527} -{"stream":"profiles","data":{"profileId":3039403378822505,"countryCode":"US","currencyCode":"USD","timezone":"America/Los_Angeles","accountInfo":{"marketplaceStringId":"ATVPDKIKX0DER","id":"ENTITY2ZP3PPFBG2043","type":"vendor","name":"3PTestBrand-A3LUQZ2NBMFGO4215634471126","validPaymentMethod":true}},"emitted_at":1659020216528} -{"stream":"sponsored_display_campaigns","data":{"campaignId":25934734632378,"name":"Campaign - 7/20/2022 15:45:46","tactic":"T00020","startDate":"20240510","state":"enabled","costType":"cpc","budget":1,"budgetType":"daily","deliveryProfile":"as_soon_as_possible"},"emitted_at":1659020217679} -{"stream":"sponsored_display_ad_groups","data":{"adGroupId":239470166910761,"campaignId":25934734632378,"defaultBid":0.02,"name":"Ad group - 7/20/2022 15:45:46","state":"enabled","bidOptimization":"clicks","tactic":"T00020"},"emitted_at":1659020218593} -{"stream":"sponsored_display_product_ads","data":{"adId":125773733335504,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT390","state":"enabled"},"emitted_at":1659020219604} -{"stream":"sponsored_display_product_ads","data":{"adId":22923447445879,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBJK","state":"enabled"},"emitted_at":1659020219605} -{"stream":"sponsored_display_product_ads","data":{"adId":174434781640143,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B006K1JR0W","state":"enabled"},"emitted_at":1659020219605} -{"stream":"sponsored_display_product_ads","data":{"adId":209576432984926,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNV58W","state":"enabled"},"emitted_at":1659020219605} -{"stream":"sponsored_display_product_ads","data":{"adId":78757678617297,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E0BD0","state":"enabled"},"emitted_at":1659020219606} -{"stream":"sponsored_display_product_ads","data":{"adId":193756923178712,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBNG","state":"enabled"},"emitted_at":1659020219606} -{"stream":"sponsored_display_product_ads","data":{"adId":31271769792588,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT38G","state":"enabled"},"emitted_at":1659020219606} -{"stream":"sponsored_display_product_ads","data":{"adId":150153237605370,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNV596","state":"enabled"},"emitted_at":1659020219606} -{"stream":"sponsored_display_product_ads","data":{"adId":2074333536480,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E0R66","state":"enabled"},"emitted_at":1659020219607} -{"stream":"sponsored_display_product_ads","data":{"adId":123533571549424,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNRD2O","state":"enabled"},"emitted_at":1659020219607} -{"stream":"sponsored_display_product_ads","data":{"adId":217260138761504,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B091FZ92NV","state":"enabled"},"emitted_at":1659020219607} -{"stream":"sponsored_display_product_ads","data":{"adId":145457886517316,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNRD1U","state":"enabled"},"emitted_at":1659020219607} -{"stream":"sponsored_display_product_ads","data":{"adId":203822232798249,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E9VEK","state":"enabled"},"emitted_at":1659020219608} -{"stream":"sponsored_display_product_ads","data":{"adId":117735697461953,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBNQ","state":"enabled"},"emitted_at":1659020219608} -{"stream":"sponsored_display_product_ads","data":{"adId":142089319699283,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B091G29WN9","state":"enabled"},"emitted_at":1659020219608} -{"stream":"sponsored_display_product_ads","data":{"adId":95431347262692,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E1JCM","state":"enabled"},"emitted_at":1659020219608} -{"stream":"sponsored_display_product_ads","data":{"adId":155014902487440,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBJU","state":"enabled"},"emitted_at":1659020219609} -{"stream":"sponsored_display_product_ads","data":{"adId":11743222321360,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT3AE","state":"enabled"},"emitted_at":1659020219609} -{"stream":"sponsored_display_product_ads","data":{"adId":103439653344998,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00RW78E52","state":"enabled"},"emitted_at":1659020219609} -{"stream":"sponsored_display_product_ads","data":{"adId":265969657657801,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT39K","state":"enabled"},"emitted_at":1659020219609} -{"stream":"sponsored_display_product_ads","data":{"adId":109412610635634,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT39U","state":"enabled"},"emitted_at":1659020219610} -{"stream":"sponsored_display_product_ads","data":{"adId":136393331771998,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNV59Q","state":"enabled"},"emitted_at":1659020219610} -{"stream":"sponsored_display_product_ads","data":{"adId":186420999434919,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNRD2E","state":"enabled"},"emitted_at":1659020219610} -{"stream":"sponsored_display_product_ads","data":{"adId":278853238562368,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B091G35ZDQ","state":"enabled"},"emitted_at":1659020219610} -{"stream":"sponsored_display_product_ads","data":{"adId":166899201791771,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E1SLE","state":"enabled"},"emitted_at":1659020219611} -{"stream":"sponsored_display_product_ads","data":{"adId":109280751164007,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B091G3QCHL","state":"enabled"},"emitted_at":1659020219611} -{"stream":"sponsored_display_product_ads","data":{"adId":151372475824008,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT39A","state":"enabled"},"emitted_at":1659020219611} -{"stream":"sponsored_display_product_ads","data":{"adId":111491538035732,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00CKZKG20","state":"enabled"},"emitted_at":1659020219611} -{"stream":"sponsored_display_product_ads","data":{"adId":61045475129398,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E3HUO","state":"enabled"},"emitted_at":1659020219611} -{"stream":"sponsored_display_product_ads","data":{"adId":125617015283672,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBO0","state":"enabled"},"emitted_at":1659020219612} -{"stream":"sponsored_display_product_ads","data":{"adId":183608040922804,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBMW","state":"enabled"},"emitted_at":1659020219612} -{"stream":"sponsored_display_product_ads","data":{"adId":252975632234287,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNV58M","state":"enabled"},"emitted_at":1659020219612} -{"stream":"sponsored_display_product_ads","data":{"adId":223374763750850,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNRD2Y","state":"enabled"},"emitted_at":1659020219612} -{"stream":"sponsored_display_product_ads","data":{"adId":155052344322362,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT37M","state":"enabled"},"emitted_at":1659020219613} -{"stream":"sponsored_display_product_ads","data":{"adId":210510170479158,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT3AY","state":"enabled"},"emitted_at":1659020219613} -{"stream":"sponsored_display_product_ads","data":{"adId":179517989169690,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT37W","state":"enabled"},"emitted_at":1659020219613} -{"stream":"sponsored_display_product_ads","data":{"adId":163992879107492,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNV5AA","state":"enabled"},"emitted_at":1659020219613} -{"stream":"sponsored_display_product_ads","data":{"adId":103527738992867,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT386","state":"enabled"},"emitted_at":1659020219614} -{"stream":"sponsored_display_product_ads","data":{"adId":195948665185008,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBOA","state":"enabled"},"emitted_at":1659020219614} -{"stream":"sponsored_display_product_ads","data":{"adId":130802512011075,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B091G1HT4P","state":"enabled"},"emitted_at":1659020219614} -{"stream":"sponsored_display_targetings","data":{"adGroupId":239470166910761,"bid":0.4,"expression":[{"type":"similarProduct"}],"expressionType":"auto","resolvedExpression":[{"type":"similarProduct"}],"state":"enabled","targetId":124150067548052},"emitted_at":1659020220625} -{"stream":"sponsored_product_campaigns","data":{"campaignId":39413387973397,"name":"Test campaging for profileId 1861552880916640","campaignType":"sponsoredProducts","targetingType":"manual","premiumBidAdjustment":true,"dailyBudget":10,"ruleBasedBudget":{"isProcessing":false},"startDate":"20220705","endDate":"20220712","state":"paused","bidding":{"strategy":"legacyForSales","adjustments":[{"predicate":"placementTop","percentage":50}]},"tags":{"PONumber":"examplePONumber","accountManager":"exampleAccountManager"}},"emitted_at":1659020221212} -{"stream":"sponsored_product_campaigns","data":{"campaignId":135264288913079,"name":"Campaign - 7/5/2022 18:14:02","campaignType":"sponsoredProducts","targetingType":"auto","premiumBidAdjustment":false,"dailyBudget":10,"startDate":"20220705","state":"enabled","bidding":{"strategy":"legacyForSales","adjustments":[]}},"emitted_at":1659020221384} -{"stream":"sponsored_product_campaigns","data":{"campaignId":191249325250025,"name":"Campaign - 7/8/2022 13:57:48","campaignType":"sponsoredProducts","targetingType":"auto","premiumBidAdjustment":true,"dailyBudget":50,"startDate":"20220708","state":"enabled","bidding":{"strategy":"legacyForSales","adjustments":[{"predicate":"placementTop","percentage":100},{"predicate":"placementProductPage","percentage":100}]}},"emitted_at":1659020221384} -{"stream":"sponsored_product_campaigns","data":{"campaignId":146003174711486,"name":"Test campaging for profileId 3039403378822505","campaignType":"sponsoredProducts","targetingType":"manual","premiumBidAdjustment":true,"dailyBudget":1,"startDate":"20220705","endDate":"20231111","state":"enabled","bidding":{"strategy":"legacyForSales","adjustments":[{"predicate":"placementTop","percentage":50}]},"tags":{"PONumber":"examplePONumber","accountManager":"exampleAccountManager"}},"emitted_at":1659020221553} -{"stream":"sponsored_product_ad_groups","data":{"adGroupId":226404883721634,"name":"My AdGroup for Campaign 39413387973397","campaignId":39413387973397,"defaultBid":10,"state":"enabled"},"emitted_at":1659020222108} -{"stream":"sponsored_product_ad_groups","data":{"adGroupId":183961953969922,"name":"Ad group - 7/5/2022 18:14:02","campaignId":135264288913079,"defaultBid":0.75,"state":"enabled"},"emitted_at":1659020222276} -{"stream":"sponsored_product_ad_groups","data":{"adGroupId":108551155050351,"name":"Ad group - 7/8/2022 13:57:48","campaignId":191249325250025,"defaultBid":1,"state":"enabled"},"emitted_at":1659020222276} -{"stream":"sponsored_product_ad_groups","data":{"adGroupId":103188883625219,"name":"My AdGroup for Campaign 146003174711486","campaignId":146003174711486,"defaultBid":10,"state":"enabled"},"emitted_at":1659020222593} -{"stream":"sponsored_product_keywords","data":{"keywordId":88368653576677,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"keyword1","matchType":"exact","state":"enabled","bid":1.12},"emitted_at":1659020223173} -{"stream":"sponsored_product_keywords","data":{"keywordId":256414981667762,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"test book","matchType":"broad","state":"enabled","bid":1.12},"emitted_at":1659020223174} -{"stream":"sponsored_product_keywords","data":{"keywordId":162522197737998,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"test book","matchType":"phrase","state":"enabled","bid":2.85},"emitted_at":1659020223175} -{"stream":"sponsored_product_keywords","data":{"keywordId":156474025571250,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"test book","matchType":"exact","state":"enabled","bid":1.12},"emitted_at":1659020223175} -{"stream":"sponsored_product_keywords","data":{"keywordId":97960974522677,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"exam book","matchType":"broad","state":"enabled","bid":0.83},"emitted_at":1659020223175} -{"stream":"sponsored_product_keywords","data":{"keywordId":21494218191267,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"exam book","matchType":"phrase","state":"enabled","bid":4.06},"emitted_at":1659020223175} -{"stream":"sponsored_product_keywords","data":{"keywordId":122265145299463,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"exam book","matchType":"exact","state":"enabled","bid":1.12},"emitted_at":1659020223176} -{"stream":"sponsored_product_keywords","data":{"keywordId":105707339702386,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"smartphone","matchType":"broad","state":"enabled","bid":3.52},"emitted_at":1659020223176} -{"stream":"sponsored_product_keywords","data":{"keywordId":185938124401124,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"smartphone","matchType":"phrase","state":"enabled","bid":3.44},"emitted_at":1659020223176} -{"stream":"sponsored_product_keywords","data":{"keywordId":16455263285469,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"smartphone","matchType":"exact","state":"enabled","bid":3.69},"emitted_at":1659020223177} -{"stream":"sponsored_product_negative_keywords","data":{"keywordId":32531566025493,"adGroupId":226404883721634,"campaignId":39413387973397,"keywordText":"negkeyword1","matchType":"negativeExact","state":"enabled"},"emitted_at":1659020224091} -{"stream":"sponsored_product_ads","data":{"adId":134721479349712,"adGroupId":226404883721634,"campaignId":39413387973397,"asin":"B09X3NTQ5S","state":"enabled"},"emitted_at":1659020225056} -{"stream":"sponsored_product_ads","data":{"adId":265970953521535,"adGroupId":226404883721634,"campaignId":39413387973397,"asin":"B09X3QCS24","state":"enabled"},"emitted_at":1659020225057} -{"stream":"sponsored_product_ads","data":{"adId":253366527049144,"adGroupId":226404883721634,"campaignId":39413387973397,"asin":"B09X3P7D6Z","state":"enabled"},"emitted_at":1659020225057} -{"stream":"sponsored_product_ads","data":{"adId":44137758141732,"adGroupId":183961953969922,"campaignId":135264288913079,"asin":"B000VHYM2E","sku":"0R-4KDA-Z2U8","state":"enabled"},"emitted_at":1659020225248} -{"stream":"sponsored_product_ads","data":{"adId":126456292487945,"adGroupId":108551155050351,"campaignId":191249325250025,"asin":"B074K5MDLW","sku":"2J-D6V7-C8XI","state":"enabled"},"emitted_at":1659020225248} -{"stream":"sponsored_product_ads","data":{"adId":125773733335504,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT390","state":"enabled"},"emitted_at":1659020225461} -{"stream":"sponsored_product_ads","data":{"adId":22923447445879,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBJK","state":"enabled"},"emitted_at":1659020225461} -{"stream":"sponsored_product_ads","data":{"adId":174434781640143,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B006K1JR0W","state":"enabled"},"emitted_at":1659020225462} -{"stream":"sponsored_product_ads","data":{"adId":209576432984926,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNV58W","state":"enabled"},"emitted_at":1659020225462} -{"stream":"sponsored_product_ads","data":{"adId":78757678617297,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E0BD0","state":"enabled"},"emitted_at":1659020225462} -{"stream":"sponsored_product_ads","data":{"adId":193756923178712,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBNG","state":"enabled"},"emitted_at":1659020225462} -{"stream":"sponsored_product_ads","data":{"adId":31271769792588,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT38G","state":"enabled"},"emitted_at":1659020225463} -{"stream":"sponsored_product_ads","data":{"adId":150153237605370,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNV596","state":"enabled"},"emitted_at":1659020225463} -{"stream":"sponsored_product_ads","data":{"adId":2074333536480,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E0R66","state":"enabled"},"emitted_at":1659020225463} -{"stream":"sponsored_product_ads","data":{"adId":123533571549424,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNRD2O","state":"enabled"},"emitted_at":1659020225463} -{"stream":"sponsored_product_ads","data":{"adId":217260138761504,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B091FZ92NV","state":"enabled"},"emitted_at":1659020225464} -{"stream":"sponsored_product_ads","data":{"adId":145457886517316,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNRD1U","state":"enabled"},"emitted_at":1659020225464} -{"stream":"sponsored_product_ads","data":{"adId":203822232798249,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E9VEK","state":"enabled"},"emitted_at":1659020225464} -{"stream":"sponsored_product_ads","data":{"adId":117735697461953,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBNQ","state":"enabled"},"emitted_at":1659020225464} -{"stream":"sponsored_product_ads","data":{"adId":142089319699283,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B091G29WN9","state":"enabled"},"emitted_at":1659020225465} -{"stream":"sponsored_product_ads","data":{"adId":95431347262692,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E1JCM","state":"enabled"},"emitted_at":1659020225465} -{"stream":"sponsored_product_ads","data":{"adId":155014902487440,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBJU","state":"enabled"},"emitted_at":1659020225465} -{"stream":"sponsored_product_ads","data":{"adId":11743222321360,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT3AE","state":"enabled"},"emitted_at":1659020225465} -{"stream":"sponsored_product_ads","data":{"adId":103439653344998,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00RW78E52","state":"enabled"},"emitted_at":1659020225466} -{"stream":"sponsored_product_ads","data":{"adId":265969657657801,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT39K","state":"enabled"},"emitted_at":1659020225466} -{"stream":"sponsored_product_ads","data":{"adId":109412610635634,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT39U","state":"enabled"},"emitted_at":1659020225466} -{"stream":"sponsored_product_ads","data":{"adId":136393331771998,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNV59Q","state":"enabled"},"emitted_at":1659020225466} -{"stream":"sponsored_product_ads","data":{"adId":186420999434919,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNRD2E","state":"enabled"},"emitted_at":1659020225467} -{"stream":"sponsored_product_ads","data":{"adId":278853238562368,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B091G35ZDQ","state":"enabled"},"emitted_at":1659020225467} -{"stream":"sponsored_product_ads","data":{"adId":166899201791771,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E1SLE","state":"enabled"},"emitted_at":1659020225467} -{"stream":"sponsored_product_ads","data":{"adId":109280751164007,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B091G3QCHL","state":"enabled"},"emitted_at":1659020225467} -{"stream":"sponsored_product_ads","data":{"adId":151372475824008,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT39A","state":"enabled"},"emitted_at":1659020225467} -{"stream":"sponsored_product_ads","data":{"adId":111491538035732,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00CKZKG20","state":"enabled"},"emitted_at":1659020225468} -{"stream":"sponsored_product_ads","data":{"adId":61045475129398,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B00U2E3HUO","state":"enabled"},"emitted_at":1659020225468} -{"stream":"sponsored_product_ads","data":{"adId":125617015283672,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBO0","state":"enabled"},"emitted_at":1659020225468} -{"stream":"sponsored_product_ads","data":{"adId":183608040922804,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBMW","state":"enabled"},"emitted_at":1659020225468} -{"stream":"sponsored_product_ads","data":{"adId":252975632234287,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNV58M","state":"enabled"},"emitted_at":1659020225469} -{"stream":"sponsored_product_ads","data":{"adId":223374763750850,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNRD2Y","state":"enabled"},"emitted_at":1659020225469} -{"stream":"sponsored_product_ads","data":{"adId":155052344322362,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT37M","state":"enabled"},"emitted_at":1659020225469} -{"stream":"sponsored_product_ads","data":{"adId":210510170479158,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT3AY","state":"enabled"},"emitted_at":1659020225470} -{"stream":"sponsored_product_ads","data":{"adId":179517989169690,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT37W","state":"enabled"},"emitted_at":1659020225470} -{"stream":"sponsored_product_ads","data":{"adId":163992879107492,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNV5AA","state":"enabled"},"emitted_at":1659020225470} -{"stream":"sponsored_product_ads","data":{"adId":103527738992867,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT386","state":"enabled"},"emitted_at":1659020225470} -{"stream":"sponsored_product_ads","data":{"adId":195948665185008,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBOA","state":"enabled"},"emitted_at":1659020225470} -{"stream":"sponsored_product_ads","data":{"adId":130802512011075,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B091G1HT4P","state":"enabled"},"emitted_at":1659020225471} -{"stream":"sponsored_product_targetings","data":{"targetId":50319181484813,"adGroupId":183961953969922,"campaignId":135264288913079,"expressionType":"auto","state":"enabled","expression":[{"type":"queryHighRelMatches"}],"resolvedExpression":[{"type":"queryHighRelMatches"}]},"emitted_at":1659020226434} -{"stream":"sponsored_product_targetings","data":{"targetId":27674318672023,"adGroupId":183961953969922,"campaignId":135264288913079,"expressionType":"auto","state":"enabled","expression":[{"type":"queryBroadRelMatches"}],"resolvedExpression":[{"type":"queryBroadRelMatches"}]},"emitted_at":1659020226435} -{"stream":"sponsored_product_targetings","data":{"targetId":231060819625654,"adGroupId":183961953969922,"campaignId":135264288913079,"expressionType":"auto","state":"enabled","expression":[{"type":"asinAccessoryRelated"}],"resolvedExpression":[{"type":"asinAccessoryRelated"}]},"emitted_at":1659020226435} -{"stream":"sponsored_product_targetings","data":{"targetId":223980840024498,"adGroupId":183961953969922,"campaignId":135264288913079,"expressionType":"auto","state":"enabled","expression":[{"type":"asinSubstituteRelated"}],"resolvedExpression":[{"type":"asinSubstituteRelated"}]},"emitted_at":1659020226436} -{"stream":"sponsored_product_targetings","data":{"targetId":62579800516352,"adGroupId":108551155050351,"campaignId":191249325250025,"expressionType":"auto","state":"enabled","expression":[{"type":"queryHighRelMatches"}],"resolvedExpression":[{"type":"queryHighRelMatches"}]},"emitted_at":1659020226436} -{"stream":"sponsored_product_targetings","data":{"targetId":232221427954900,"adGroupId":108551155050351,"campaignId":191249325250025,"expressionType":"auto","state":"enabled","expression":[{"type":"queryBroadRelMatches"}],"resolvedExpression":[{"type":"queryBroadRelMatches"}]},"emitted_at":1659020226436} -{"stream":"sponsored_product_targetings","data":{"targetId":12739477778779,"adGroupId":108551155050351,"campaignId":191249325250025,"expressionType":"auto","state":"enabled","expression":[{"type":"asinAccessoryRelated"}],"resolvedExpression":[{"type":"asinAccessoryRelated"}]},"emitted_at":1659020226436} -{"stream":"sponsored_product_targetings","data":{"targetId":1189452552122,"adGroupId":108551155050351,"campaignId":191249325250025,"expressionType":"auto","state":"enabled","expression":[{"type":"asinSubstituteRelated"}],"resolvedExpression":[{"type":"asinSubstituteRelated"}]},"emitted_at":1659020226437} diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/__init__.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/__init__.py index a578024d6d9c..f5a7f3532468 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/__init__.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/__init__.py @@ -21,6 +21,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # +from .attribution_report import AttributionReportModel from .common import CatalogModel, Keywords, MetricsReport, NegativeKeywords from .profile import Profile from .sponsored_brands import BrandsAdGroup, BrandsCampaign @@ -43,4 +44,5 @@ "ProductCampaign", "ProductTargeting", "Profile", + "AttributionReportModel", ] diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/attribution_report.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/attribution_report.py new file mode 100644 index 000000000000..e9dd0e67c48f --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/attribution_report.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from typing import List + +from .common import CatalogModel + + +class Report(CatalogModel): + date: str + brandName: str + marketplace: str + campaignId: str + productAsin: str + productConversionType: str + advertiserName: str + adGroupId: str + creativeId: str + productName: str + productCategory: str + productSubcategory: str + productGroup: str + publisher: str + + +class AttributionReportModel(CatalogModel): + reports: List[Report] + size: int + cursorId: str diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py index a03c781355e3..5810971db1d5 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py @@ -13,6 +13,10 @@ from .schemas import Profile from .streams import ( + AttributionReportPerformanceAdgroup, + AttributionReportPerformanceCampaign, + AttributionReportPerformanceCreative, + AttributionReportProducts, Profiles, SponsoredBrandsAdGroups, SponsoredBrandsCampaigns, @@ -102,6 +106,10 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: SponsoredBrandsKeywords, SponsoredBrandsReportStream, SponsoredBrandsVideoReportStream, + AttributionReportPerformanceAdgroup, + AttributionReportPerformanceCampaign, + AttributionReportPerformanceCreative, + AttributionReportProducts, ] return [profiles_stream, *[stream_class(**stream_args) for stream_class in non_profile_stream_classes]] diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/__init__.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/__init__.py index 2525ffa63579..1d7ab84d65df 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/__init__.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/__init__.py @@ -1,6 +1,12 @@ # # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # +from .attribution_report import ( + AttributionReportPerformanceAdgroup, + AttributionReportPerformanceCampaign, + AttributionReportPerformanceCreative, + AttributionReportProducts, +) from .profiles import Profiles from .report_streams import ( SponsoredBrandsReportStream, @@ -38,4 +44,8 @@ "SponsoredProductsReportStream", "SponsoredBrandsReportStream", "SponsoredBrandsVideoReportStream", + "AttributionReportPerformanceAdgroup", + "AttributionReportPerformanceCampaign", + "AttributionReportPerformanceCreative", + "AttributionReportProducts", ] diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/attribution_report.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/attribution_report.py new file mode 100644 index 000000000000..694564038c50 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/attribution_report.py @@ -0,0 +1,175 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Iterable, Mapping, MutableMapping, Optional + +import pendulum +import requests +from source_amazon_ads.schemas import AttributionReportModel, Profile +from source_amazon_ads.streams.common import AmazonAdsStream + +BRAND_REFERRAL_BONUS = "brb_bonus_amount" + +METRICS_MAP = { + "PERFORMANCE": [ + "Click-throughs", + "attributedDetailPageViewsClicks14d", + "attributedAddToCartClicks14d", + "attributedPurchases14d", + "unitsSold14d", + "attributedSales14d", + "attributedTotalDetailPageViewsClicks14d", + "attributedTotalAddToCartClicks14d", + "attributedTotalPurchases14d", + "totalUnitsSold14d", + "totalAttributedSales14d", + ], + "PRODUCTS": [ + "attributedDetailPageViewsClicks14d", + "attributedAddToCartClicks14d", + "attributedPurchases14d", + "unitsSold14d", + "attributedSales14d", + "brandHaloDetailPageViewsClicks14d", + "brandHaloAttributedAddToCartClicks14d", + "brandHaloAttributedPurchases14d", + "brandHaloUnitsSold14d", + "brandHaloAttributedSales14d", + "attributedNewToBrandPurchases14d", + "attributedNewToBrandUnitsSold14d", + "attributedNewToBrandSales14d", + "brandHaloNewToBrandPurchases14d", + "brandHaloNewToBrandUnitsSold14d", + "brandHaloNewToBrandSales14d", + ], +} + + +class AttributionReport(AmazonAdsStream): + """ + This stream corresponds to Amazon Advertising API - Attribution Reports + https://advertising.amazon.com/API/docs/en-us/amazon-attribution-prod-3p/#/ + """ + + model = AttributionReportModel + primary_key = None + data_field = "reports" + page_size = 300 + + report_type = "" + metrics = "" + group_by = "" + + _next_page_token_field = "cursorId" + _current_profile_id = "" + + REPORT_DATE_FORMAT = "YYYYMMDD" + CONFIG_DATE_FORMAT = "YYYY-MM-DD" + REPORTING_PERIOD = 90 + + def __init__(self, config: Mapping[str, Any], *args, **kwargs): + self._start_date = config.get("start_date") + self._req_start_date = "" + self._req_end_date = "" + + super().__init__(config, *args, **kwargs) + + def _set_dates(self, profile: Profile): + new_start_date = pendulum.now(tz=profile.timezone).subtract(days=1).date() + new_end_date = pendulum.now(tz=profile.timezone).date() + + if self._start_date: + new_start_date = max(self._start_date, new_end_date.subtract(days=self.REPORTING_PERIOD)) + + self._req_start_date = new_start_date.format(self.REPORT_DATE_FORMAT) + self._req_end_date = new_end_date.format(self.REPORT_DATE_FORMAT) + + @property + def http_method(self) -> str: + return "POST" + + def read_records(self, *args, **kvargs) -> Iterable[Mapping[str, Any]]: + """ + Iterate through self._profiles list and send read all records for each profile. + """ + for profile in self._profiles: + try: + self._set_dates(profile) + self._current_profile_id = profile.profileId + yield from super().read_records(*args, **kvargs) + except Exception as err: + self.logger.info("some error occurred: %s", err) + + def request_headers(self, *args, **kvargs) -> MutableMapping[str, Any]: + headers = super().request_headers(*args, **kvargs) + headers["Amazon-Advertising-API-Scope"] = str(self._current_profile_id) + return headers + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + stream_data = response.json() + next_page_token = stream_data.get(self._next_page_token_field) + if next_page_token: + return {self._next_page_token_field: next_page_token} + + def path(self, **kvargs) -> str: + return "/attribution/report" + + def request_body_json( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Optional[Mapping]: + body = { + "reportType": self.report_type, + "count": self.page_size, + "metrics": self.metrics, + "startDate": self._req_start_date, + "endDate": self._req_end_date, + self._next_page_token_field: "", + } + + if self.group_by: + body["groupBy"] = self.group_by + + if next_page_token: + body[self._next_page_token_field] = next_page_token[self._next_page_token_field] + + return body + + +class AttributionReportProducts(AttributionReport): + report_type = "PRODUCTS" + + metrics = ",".join(METRICS_MAP[report_type]) + + group_by = "" + + +class AttributionReportPerformanceCreative(AttributionReport): + report_type = "PERFORMANCE" + + metrics = ",".join(METRICS_MAP[report_type]) + + group_by = "CREATIVE" + + +class AttributionReportPerformanceAdgroup(AttributionReport): + report_type = "PERFORMANCE" + + metrics_list = METRICS_MAP[report_type] + metrics_list.append(BRAND_REFERRAL_BONUS) + metrics = ",".join(metrics_list) + + group_by = "ADGROUP" + + +class AttributionReportPerformanceCampaign(AttributionReport): + report_type = "PERFORMANCE" + + metrics_list = METRICS_MAP[report_type] + metrics_list.append(BRAND_REFERRAL_BONUS) + metrics = ",".join(metrics_list) + + group_by = "CAMPAIGN" diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py index 86f1647668ff..b955e18ba07d 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py @@ -97,6 +97,8 @@ class AmazonAdsStream(HttpStream, BasicAmazonAdsStream): Class for getting data from streams that based on single http request. """ + data_field = "" + def __init__(self, config: Mapping[str, Any], *args, profiles: List[Profile] = None, **kwargs): # Each AmazonAdsStream instance are dependant on list of profiles. BasicAmazonAdsStream.__init__(self, config, profiles=profiles) @@ -121,7 +123,10 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp :return an object representing single record in the response """ if response.status_code == HTTPStatus.OK: - yield from response.json() + if self.data_field: + yield from response.json().get(self.data_field, []) + else: + yield from response.json() return """ diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py index 54cfe29dfefb..6cf2617a71c2 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py @@ -2,6 +2,7 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +import json from copy import deepcopy from pytest import fixture @@ -61,3 +62,121 @@ def targeting_response(): return """ [{"targetId":123,"adGroupId":321,"state":"enabled","expressionType":"manual","bid":1.5,"expression":{"type":"asinSameAs","value":"B0123456789"},"resolvedExpression":{"type":"views","values":{"type":"asinCategorySameAs","value":"B0123456789"}}}] """ + + +@fixture +def attribution_report_response(): + def _internal(report_type: str): + responses = { + "PRODUCTS": { + "reports": [ + { + "date": "20220829", + "attributedDetailPageViewsClicks14d": "0", + "attributedPurchases14d": "0", + "adGroupId": "bestselling_fan-dusters", + "advertiserName": "name", + "productName": "some product name", + "productCategory": "Chemicals", + "productSubcategory": "Applicators", + "brandHaloAttributedPurchases14d": "0", + "brandHaloUnitsSold14d": "0", + "attributedNewToBrandSales14d": "0", + "attributedAddToCartClicks14d": "0", + "brandHaloNewToBrandPurchases14d": "0", + "brandName": "name", + "marketplace": "AMAZON.COM", + "brandHaloAttributedSales14d": "0", + "campaignId": "my-campaign", + "brandHaloNewToBrandUnitsSold14d": "0", + "productAsin": "AAAAAAA", + "productConversionType": "Brand Halo", + "attributedNewToBrandUnitsSold14d": "0", + "brandHaloAttributedAddToCartClicks14d": "0", + "attributedNewToBrandPurchases14d": "0", + "unitsSold14d": "0", + "productGroup": "Automotive", + "brandHaloNewToBrandSales14d": "0", + "publisher": "Display - Other", + "brandHaloDetailPageViewsClicks14d": "0", + "attributedSales14d": "0", + } + ] + }, + "PERFORMANCE_ADGROUP": { + "reports": [ + { + "date": "20220829", + "attributedAddToCartClicks14d": "5", + "brb_bonus_amount": "14.280000000000001", + "campaignId": "16719043411", + "attributedDetailPageViewsClicks14d": "30", + "attributedPurchases14d": "3", + "attributedTotalAddToCartClicks14d": "5", + "attributedTotalPurchases14d": "3", + "adGroupId": "135021988277", + "advertiserName": "Eversprout", + "totalUnitsSold14d": "4", + "unitsSold14d": "4", + "Click-throughs": "30", + "publisher": "Google Ads", + "attributedTotalDetailPageViewsClicks14d": "30", + "attributedSales14d": "191.95999999999998", + "totalAttributedSales14d": "191.95999999999998", + } + ] + }, + "PERFORMANCE_CAMPAIGN": { + "reports": [ + { + "date": "20220830", + "attributedAddToCartClicks14d": "1", + "brb_bonus_amount": "0", + "campaignId": "3936789099315437-B082P9Y919", + "attributedDetailPageViewsClicks14d": "9", + "attributedPurchases14d": "0", + "attributedTotalAddToCartClicks14d": "1", + "attributedTotalPurchases14d": "0", + "advertiserName": "Eversprout", + "totalUnitsSold14d": "0", + "unitsSold14d": "0", + "Click-throughs": "12", + "attributedTotalDetailPageViewsClicks14d": "16", + "attributedSales14d": "0", + "totalAttributedSales14d": "0", + } + ] + }, + "PERFORMANCE_CREATIVE": { + "reports": [ + { + "date": "20220830", + "attributedAddToCartClicks14d": "0", + "campaignId": "16719043411", + "attributedDetailPageViewsClicks14d": "0", + "attributedPurchases14d": "0", + "attributedTotalAddToCartClicks14d": "0", + "attributedTotalPurchases14d": "0", + "adGroupId": "135021988277", + "advertiserName": "Eversprout", + "creativeId": "135021988277", + "totalUnitsSold14d": "0", + "unitsSold14d": "0", + "Click-throughs": "1", + "publisher": "Google Ads", + "attributedTotalDetailPageViewsClicks14d": "0", + "attributedSales14d": "0", + "totalAttributedSales14d": "0", + } + ] + }, + } + + return json.dumps(responses[report_type]) + + return _internal + + +@fixture +def attribution_report_bad_response(): + return "bad response" diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_attribution_report.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_attribution_report.py new file mode 100644 index 000000000000..1761be5913f7 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_attribution_report.py @@ -0,0 +1,148 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import json + +import pytest +import requests +import responses +from airbyte_cdk.models import SyncMode +from jsonschema import validate +from source_amazon_ads import SourceAmazonAds + + +def setup_responses( + profiles_response=None, + attribution_report_response=None, +): + responses.add( + responses.POST, + "https://api.amazon.com/auth/o2/token", + json={"access_token": "alala", "expires_in": 10}, + ) + if profiles_response: + responses.add( + responses.GET, + "https://advertising-api.amazon.com/v2/profiles", + body=profiles_response, + ) + if attribution_report_response: + responses.add( + responses.POST, + "https://advertising-api.amazon.com/attribution/report", + body=attribution_report_response, + ) + + +def get_all_stream_records(stream): + records = stream.read_records(SyncMode.full_refresh) + return [r for r in records] + + +def get_stream_by_name(streams, stream_name): + for stream in streams: + if stream.name == stream_name: + return stream + raise Exception(f"Expected stream {stream_name} not found") + + +@pytest.mark.parametrize( + ("stream_name", "report_type"), + [ + ("attribution_report_products", "PRODUCTS"), + ("attribution_report_performance_adgroup", "PERFORMANCE_ADGROUP"), + ("attribution_report_performance_campaign", "PERFORMANCE_CAMPAIGN"), + ("attribution_report_performance_creative", "PERFORMANCE_CREATIVE"), + ], +) +@responses.activate +def test_attribution_report_schema(config, profiles_response, attribution_report_response, stream_name, report_type): + # custom start date + config["start_date"] = "2022-09-03" + + setup_responses(profiles_response=profiles_response, attribution_report_response=attribution_report_response(report_type)) + + source = SourceAmazonAds() + streams = source.streams(config) + + profile_stream = get_stream_by_name(streams, "profiles") + attribution_report_stream = get_stream_by_name(streams, stream_name) + schema = attribution_report_stream.get_json_schema() + + profile_records = get_all_stream_records(profile_stream) + attribution_records = get_all_stream_records(attribution_report_stream) + assert len(attribution_records) == len(profile_records) * len(json.loads(attribution_report_response(report_type)).get("reports")) + + for record in attribution_records: + validate(schema=schema, instance=record) + + +@pytest.mark.parametrize( + ("stream_name", "report_type"), + [ + ("attribution_report_products", "PRODUCTS"), + ("attribution_report_performance_adgroup", "PERFORMANCE_ADGROUP"), + ("attribution_report_performance_campaign", "PERFORMANCE_CAMPAIGN"), + ("attribution_report_performance_creative", "PERFORMANCE_CREATIVE"), + ], +) +@responses.activate +def test_attribution_report_with_pagination(mocker, config, profiles_response, attribution_report_response, stream_name, report_type): + profiles = json.loads(profiles_response) + # use only single profile + profiles_response = json.dumps([profiles[0]]) + + setup_responses(profiles_response=profiles_response) + + source = SourceAmazonAds() + streams = source.streams(config) + + attribution_report_stream = get_stream_by_name(streams, stream_name) + attribution_data = json.loads(attribution_report_response(report_type)) + + def _callback(request: requests.PreparedRequest): + attribution_data["cursorId"] = None + request_data = json.loads(request.body) + + if request_data["count"] > 0: + mocker.patch("source_amazon_ads.streams.attribution_report.AttributionReport.page_size", 0) + attribution_data["cursorId"] = "next_page_token" + + return 200, {}, json.dumps(attribution_data) + + responses.add_callback( + responses.POST, + "https://advertising-api.amazon.com/attribution/report", + content_type="application/json", + callback=_callback, + ) + + attribution_records = get_all_stream_records(attribution_report_stream) + + # request should be called 2 times for a single profile + assert len(attribution_records) == 2 * len(attribution_data.get("reports")) + + +@pytest.mark.parametrize( + ("stream_name", "report_type"), + [ + ("attribution_report_products", "PRODUCTS"), + ("attribution_report_performance_adgroup", "PERFORMANCE_ADGROUP"), + ("attribution_report_performance_campaign", "PERFORMANCE_CAMPAIGN"), + ("attribution_report_performance_creative", "PERFORMANCE_CREATIVE"), + ], +) +@responses.activate +def test_attribution_report_no_data(config, profiles_response, attribution_report_bad_response, stream_name, report_type): + # custom start date + config["start_date"] = "2022-09-03" + + setup_responses(profiles_response=profiles_response, attribution_report_response=attribution_report_bad_response) + + source = SourceAmazonAds() + streams = source.streams(config) + + attribution_report_stream = get_stream_by_name(streams, stream_name) + attribution_records = get_all_stream_records(attribution_report_stream) + assert len(attribution_records) == 0 diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py index 4637ad670fa4..5a28cf92975e 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py @@ -74,7 +74,7 @@ def test_source_streams(config): setup_responses() source = SourceAmazonAds() streams = source.streams(config) - assert len(streams) == 18 + assert len(streams) == 22 actual_stream_names = {stream.name for stream in streams} expected_stream_names = set( [ @@ -91,6 +91,10 @@ def test_source_streams(config): "sponsored_brands_ad_groups", "sponsored_brands_keywords", "sponsored_brands_report_stream", + "attribution_report_performance_adgroup", + "attribution_report_performance_campaign", + "attribution_report_performance_creative", + "attribution_report_products", ] ) assert not expected_stream_names - actual_stream_names diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md index 0218aa3b5981..00bd74d74e9a 100644 --- a/docs/integrations/sources/amazon-ads.md +++ b/docs/integrations/sources/amazon-ads.md @@ -66,6 +66,7 @@ This source is capable of syncing the following streams: * [Brand Video Reports](https://advertising.amazon.com/API/docs/en-us/reference/sponsored-brands/2/reports) * [Display Reports](https://advertising.amazon.com/API/docs/en-us/sponsored-display/3-0/openapi#/Reports) * [Products Reports](https://advertising.amazon.com/API/docs/en-us/sponsored-products/2-0/openapi#/Reports) +* [Attribution Reports](https://advertising.amazon.com/API/docs/en-us/amazon-attribution-prod-3p/#/) ## Connector-specific features and highlights @@ -89,7 +90,8 @@ Information about expected report generation waiting time you may find [here](ht ## CHANGELOG | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------- | +|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| 0.1.23 | 2022-09-06 | [16342](https://github.com/airbytehq/airbyte/pull/16342) | Add attribution reports | | 0.1.22 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state. | | 0.1.21 | 2022-09-27 | [17202](https://github.com/airbytehq/airbyte/pull/17202) | Improved handling if known reporting errors | | 0.1.20 | 2022-09-08 | [16453](https://github.com/airbytehq/airbyte/pull/16453) | Increase `report_wait_timeout` 30 -> 60 minutes |