-
Notifications
You must be signed in to change notification settings - Fork 4.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
🎉 Source Amazon Seller Partner: Add new streams (#13604)
* add new streams * update connector version in Dockerfile and docs * fix end_date if is None and save reportid in unique records * WIP: add test for slices in settlement reports * auto-bump connector version * fix: formatting Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com> Co-authored-by: Harshith Mullapudi <harshithmullapudi@gmail.com>
- Loading branch information
3 people
authored
Jul 15, 2022
1 parent
25ee6f6
commit 00c3ec7
Showing
9 changed files
with
279 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
21 changes: 21 additions & 0 deletions
21
...rtner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| { | ||
| "title": "FBA customer returns", | ||
| "description": "FBA customer returns report Data Reports", | ||
| "type": "object", | ||
| "$schema": "http://json-schema.org/draft-07/schema#", | ||
| "properties": { | ||
| "return-date": { "type": ["null", "string"] }, | ||
| "order-id": { "type": ["null", "string"] }, | ||
| "sku": { "type": ["null", "string"] }, | ||
| "asin": { "type": ["null", "string"] }, | ||
| "fnsku": { "type": ["null", "string"] }, | ||
| "product-name": { "type": ["null", "string"] }, | ||
| "quantity": { "type": ["null", "string"] }, | ||
| "fulfillment-center-id": { "type": ["null", "string"] }, | ||
| "detailed-disposition": { "type": ["null", "string"] }, | ||
| "reason": { "type": ["null", "string"] }, | ||
| "status": { "type": ["null", "string"] }, | ||
| "license-plate-number": { "type": ["null", "string"] }, | ||
| "customer-comments": { "type": ["null", "string"] } | ||
| } | ||
| } |
45 changes: 45 additions & 0 deletions
45
...partner/source_amazon_seller_partner/schemas/GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| { | ||
| "title": "Flat File Settlement Reports", | ||
| "description": "", | ||
| "type": "object", | ||
| "$schema": "http://json-schema.org/draft-07/schema#", | ||
| "properties": { | ||
| "settlement-id": { "type": ["null", "string"] }, | ||
| "settlement-start-date": { "type": ["null", "string"] }, | ||
| "settlement-end-date": { "type": ["null", "string"] }, | ||
| "deposit-date": { "type": ["null", "string"] }, | ||
| "total-amount": { "type": ["null", "string"] }, | ||
| "currency": { "type": ["null", "string"] }, | ||
| "transaction-type": { "type": ["null", "string"] }, | ||
| "order-id": { "type": ["null", "string"] }, | ||
| "merchant-order-id": { "type": ["null", "string"] }, | ||
| "adjustment-id": { "type": ["null", "string"] }, | ||
| "shipment-id": { "type": ["null", "string"] }, | ||
| "marketplace-name": { "type": ["null", "string"] }, | ||
| "shipment-fee-type": { "type": ["null", "string"] }, | ||
| "shipment-fee-amount": { "type": ["null", "string"] }, | ||
| "order-fee-type": { "type": ["null", "string"] }, | ||
| "order-fee-amount": { "type": ["null", "string"] }, | ||
| "fulfillment-id": { "type": ["null", "string"] }, | ||
| "posted-date": { "type": ["null", "string"] }, | ||
| "order-item-code": { "type": ["null", "string"] }, | ||
| "merchant-order-item-id": { "type": ["null", "string"] }, | ||
| "merchant-adjustment-item-id": { "type": ["null", "string"] }, | ||
| "sku": { "type": ["null", "string"] }, | ||
| "quantity-purchased": { "type": ["null", "string"] }, | ||
| "price-type": { "type": ["null", "string"] }, | ||
| "price-amount": { "type": ["null", "string"] }, | ||
| "item-related-fee-type": { "type": ["null", "string"] }, | ||
| "item-related-fee-amount": { "type": ["null", "string"] }, | ||
| "misc-fee-amount": { "type": ["null", "string"] }, | ||
| "other-fee-amount": { "type": ["null", "string"] }, | ||
| "other-fee-reason-description": { "type": ["null", "string"] }, | ||
| "promotion-id": { "type": ["null", "string"] }, | ||
| "promotion-type": { "type": ["null", "string"] }, | ||
| "promotion-amount": { "type": ["null", "string"] }, | ||
| "direct-payment-type": { "type": ["null", "string"] }, | ||
| "direct-payment-amount": { "type": ["null", "string"] }, | ||
| "other-amount": { "type": ["null", "string"] }, | ||
| "report_id": { "type": ["null", "string"] } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
114 changes: 114 additions & 0 deletions
114
...nectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| # | ||
| # Copyright (c) 2022 Airbyte, Inc., all rights reserved. | ||
| # | ||
|
|
||
| import pytest | ||
| import requests | ||
| from airbyte_cdk.models import SyncMode | ||
| from source_amazon_seller_partner.auth import AWSSignature | ||
| from source_amazon_seller_partner.streams import FlatFileSettlementV2Reports | ||
|
|
||
| START_DATE_1 = "2022-05-25T00:00:00Z" | ||
| END_DATE_1 = "2022-05-26T00:00:00Z" | ||
|
|
||
| generated_reports_from_amazon = { | ||
| "payload": [ | ||
| { | ||
| "createdTime": "2022-07-08T10:39:31+00:00", | ||
| "dataEndTime": "2022-07-08T09:59:21+00:00", | ||
| "dataStartTime": "2022-06-27T08:01:32+00:00", | ||
| "marketplaceIds": [ | ||
| "A1F83G8C2ARO7P", | ||
| "A1PA6795UKMFR9", | ||
| "A13V1IB3VIYZZH", | ||
| "AZMDEXL2RVFNN", | ||
| "A38D8NSA03LJTC", | ||
| "A1ZFFQZ3HTUKT9", | ||
| "APJ6JRA9NG5V4", | ||
| "A1RKKUPIHCS9HS", | ||
| "A62U237T8HV6N", | ||
| "AFQLKURYRPEL8", | ||
| "A1NYP31CE519TD", | ||
| "A1805IZSGTT6HS", | ||
| "A33AVAJ2PDY3EV", | ||
| "AMEN7PMS3EDWL", | ||
| "A2NODRKZP88ZB9", | ||
| "A1C3SOZRARQ6R3", | ||
| ], | ||
| "processingEndTime": "2022-07-08T10:39:31+00:00", | ||
| "processingStartTime": "2022-07-08T10:39:31+00:00", | ||
| "processingStatus": "DONE", | ||
| "reportDocumentId": "amzn1.spdoc.1.3.0fcde1b1-a35e-4fe1-b077-38e0e9f65d63.T1SN9707N5X5IQ.0000", | ||
| "reportId": "85968019100", | ||
| "reportType": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", | ||
| }, | ||
| { | ||
| "createdTime": "2022-07-06T09:12:07+00:00", | ||
| "dataEndTime": "2022-07-06T08:38:16+00:00", | ||
| "dataStartTime": "2022-06-22T08:38:16+00:00", | ||
| "marketplaceIds": [ | ||
| "A1F83G8C2ARO7P", | ||
| "A1PA6795UKMFR9", | ||
| "A13V1IB3VIYZZH", | ||
| "AZMDEXL2RVFNN", | ||
| "A38D8NSA03LJTC", | ||
| "A1ZFFQZ3HTUKT9", | ||
| "APJ6JRA9NG5V4", | ||
| "A1RKKUPIHCS9HS", | ||
| "A62U237T8HV6N", | ||
| "AFQLKURYRPEL8", | ||
| "A1NYP31CE519TD", | ||
| "A1805IZSGTT6HS", | ||
| "A33AVAJ2PDY3EV", | ||
| "AMEN7PMS3EDWL", | ||
| "A2NODRKZP88ZB9", | ||
| "A1C3SOZRARQ6R3", | ||
| ], | ||
| "processingEndTime": "2022-07-06T09:12:07+00:00", | ||
| "processingStartTime": "2022-07-06T09:12:07+00:00", | ||
| "processingStatus": "DONE", | ||
| "reportDocumentId": "amzn1.spdoc.1.3.f7f43990-7c58-40f2-a93f-565b79a88269.T3OS2416I1AAXM.0000", | ||
| "reportId": "85948019111", | ||
| "reportType": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", | ||
| }, | ||
| ] | ||
| } | ||
|
|
||
|
|
||
| @pytest.fixture | ||
| def settlement_reports_stream(): | ||
| def _internal(start_date: str = START_DATE_1, end_date: str = END_DATE_1): | ||
| aws_signature = AWSSignature( | ||
| service="execute-api", | ||
| aws_access_key_id="AccessKeyId", | ||
| aws_secret_access_key="SecretAccessKey", | ||
| aws_session_token="SessionToken", | ||
| region="US", | ||
| ) | ||
| stream = FlatFileSettlementV2Reports( | ||
| url_base="https://test.url", | ||
| aws_signature=aws_signature, | ||
| replication_start_date=start_date, | ||
| replication_end_date=end_date, | ||
| marketplace_id="id", | ||
| authenticator=None, | ||
| period_in_days=0, | ||
| report_options=None, | ||
| max_wait_seconds=500, | ||
| ) | ||
| return stream | ||
|
|
||
| return _internal | ||
|
|
||
|
|
||
| def test_stream_slices_method(mocker, settlement_reports_stream): | ||
| response = requests.Response() | ||
| mocker.patch.object(response, "json", return_value=generated_reports_from_amazon) | ||
|
|
||
| data = response.json().get("payload", list()) | ||
|
|
||
| slices = [{"report_id": e.get("reportId")} for e in data] | ||
|
|
||
| for i in range(len(slices)): | ||
| report = settlement_reports_stream()._create_report(sync_mode=SyncMode.full_refresh, stream_slice=slices[i]) | ||
| assert report.get("reportId") == generated_reports_from_amazon.get("payload")[i].get("reportId") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters