From b3b37bf9f166f973d7d43e758c08fb7833ff4f78 Mon Sep 17 00:00:00 2001 From: Serhii Date: Tue, 12 Jul 2022 13:25:50 +0300 Subject: [PATCH 1/7] Updated API version from v9 to v11 --- .../connectors/source-google-ads/Dockerfile | 2 +- .../connectors/source-google-ads/README.md | 2 +- .../connectors/source-google-ads/setup.py | 2 +- .../source_google_ads/google_ads.py | 2 +- .../schemas/ad_group_ads.json | 6 ++-- .../source_google_ads/schemas/campaigns.json | 2 +- .../source_google_ads/streams.py | 32 +++++++++---------- docs/integrations/sources/google-ads.md | 1 + 8 files changed, 25 insertions(+), 24 deletions(-) diff --git a/airbyte-integrations/connectors/source-google-ads/Dockerfile b/airbyte-integrations/connectors/source-google-ads/Dockerfile index 32322371644a..e38d7306c33f 100644 --- a/airbyte-integrations/connectors/source-google-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-google-ads/Dockerfile @@ -13,5 +13,5 @@ COPY main.py ./ ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.42 +LABEL io.airbyte.version=0.1.43 LABEL io.airbyte.name=airbyte/source-google-ads diff --git a/airbyte-integrations/connectors/source-google-ads/README.md b/airbyte-integrations/connectors/source-google-ads/README.md index 597a4473897d..b54f46cf87fd 100644 --- a/airbyte-integrations/connectors/source-google-ads/README.md +++ b/airbyte-integrations/connectors/source-google-ads/README.md @@ -79,7 +79,7 @@ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integrat Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. First install test dependencies into your virtual environment: ``` -pip install .[tests] +pip install -e ".[tests]" ``` ### Unit Tests To run unit tests locally, from the connector directory run: diff --git a/airbyte-integrations/connectors/source-google-ads/setup.py b/airbyte-integrations/connectors/source-google-ads/setup.py index 4f0bf491da5f..59237952ce38 100644 --- a/airbyte-integrations/connectors/source-google-ads/setup.py +++ b/airbyte-integrations/connectors/source-google-ads/setup.py @@ -7,7 +7,7 @@ # pin protobuf==3.20.0 as other versions may cause problems on different architectures # (see https://github.com/airbytehq/airbyte/issues/13580) -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "google-ads==15.1.1", "protobuf==3.20.0", "pendulum"] +MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "google-ads==17.0.0", "protobuf==3.20.0", "pendulum"] TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock", "freezegun", "requests-mock"] diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py index b39d282a7924..7fb1f36dbca6 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py @@ -30,7 +30,7 @@ "geographic_report": "geographic_view", "keyword_report": "keyword_view", } -API_VERSION = "v9" +API_VERSION = "v11" class GoogleAds: diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ads.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ads.json index 3da2773a1513..a06960e32e3d 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ads.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ads.json @@ -479,13 +479,13 @@ "type": "string" } }, - "ad_group_ad.ad.video_ad.discovery.description1": { + "ad_group_ad.ad.video_ad.in_feed.description1": { "type": ["null", "string"] }, - "ad_group_ad.ad.video_ad.discovery.description2": { + "ad_group_ad.ad.video_ad.in_feed.description2": { "type": ["null", "string"] }, - "ad_group_ad.ad.video_ad.discovery.headline": { + "ad_group_ad.ad.video_ad.in_feed.headline": { "type": ["null", "string"] }, "ad_group_ad.ad.video_ad.in_stream.action_button_label": { diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaigns.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaigns.json index 96446d569103..329bf21a1ec1 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaigns.json @@ -110,7 +110,7 @@ "campaign.maximize_conversion_value.target_roas": { "type": ["null", "number"] }, - "campaign.maximize_conversions.target_cpa": { + "campaign.maximize_conversions.target_cpa_micros": { "type": ["null", "integer"] }, "campaign.name": { diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py index 7ff591a2eded..c8e8efcc1299 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py @@ -236,7 +236,7 @@ def get_query(self, stream_slice: Mapping[str, Any] = None) -> str: class Accounts(IncrementalGoogleAdsStream): """ - Accounts stream: https://developers.google.com/google-ads/api/fields/v9/customer + Accounts stream: https://developers.google.com/google-ads/api/fields/v11/customer """ primary_key = ["customer.id", "segments.date"] @@ -253,7 +253,7 @@ class ServiceAccounts(GoogleAdsStream): class Campaigns(IncrementalGoogleAdsStream): """ - Campaigns stream: https://developers.google.com/google-ads/api/fields/v9/campaign + Campaigns stream: https://developers.google.com/google-ads/api/fields/v11/campaign """ primary_key = ["campaign.id", "segments.date"] @@ -261,7 +261,7 @@ class Campaigns(IncrementalGoogleAdsStream): class CampaignLabels(GoogleAdsStream): """ - Campaign labels stream: https://developers.google.com/google-ads/api/fields/v9/campaign_label + Campaign labels stream: https://developers.google.com/google-ads/api/fields/v11/campaign_label """ # Note that this is a string type. Google doesn't return a more convenient identifier. @@ -270,7 +270,7 @@ class CampaignLabels(GoogleAdsStream): class AdGroups(IncrementalGoogleAdsStream): """ - AdGroups stream: https://developers.google.com/google-ads/api/fields/v9/ad_group + AdGroups stream: https://developers.google.com/google-ads/api/fields/v11/ad_group """ primary_key = ["ad_group.id", "segments.date"] @@ -278,7 +278,7 @@ class AdGroups(IncrementalGoogleAdsStream): class AdGroupLabels(GoogleAdsStream): """ - Ad Group Labels stream: https://developers.google.com/google-ads/api/fields/v9/ad_group_label + Ad Group Labels stream: https://developers.google.com/google-ads/api/fields/v11/ad_group_label """ # Note that this is a string type. Google doesn't return a more convenient identifier. @@ -287,7 +287,7 @@ class AdGroupLabels(GoogleAdsStream): class AdGroupAds(IncrementalGoogleAdsStream): """ - AdGroups stream: https://developers.google.com/google-ads/api/fields/v9/ad_group_ad + AdGroups stream: https://developers.google.com/google-ads/api/fields/v11/ad_group_ad """ primary_key = ["ad_group_ad.ad.id", "segments.date"] @@ -295,7 +295,7 @@ class AdGroupAds(IncrementalGoogleAdsStream): class AdGroupAdLabels(GoogleAdsStream): """ - Ad Group Ad Labels stream: https://developers.google.com/google-ads/api/fields/v9/ad_group_ad_label + Ad Group Ad Labels stream: https://developers.google.com/google-ads/api/fields/v11/ad_group_ad_label """ # Note that this is a string type. Google doesn't return a more convenient identifier. @@ -304,61 +304,61 @@ class AdGroupAdLabels(GoogleAdsStream): class AccountPerformanceReport(IncrementalGoogleAdsStream): """ - AccountPerformanceReport stream: https://developers.google.com/google-ads/api/fields/v9/customer + AccountPerformanceReport stream: https://developers.google.com/google-ads/api/fields/v11/customer Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#account_performance """ class AdGroupAdReport(IncrementalGoogleAdsStream): """ - AdGroupAdReport stream: https://developers.google.com/google-ads/api/fields/v9/ad_group_ad + AdGroupAdReport stream: https://developers.google.com/google-ads/api/fields/v11/ad_group_ad Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#ad_performance """ class DisplayKeywordPerformanceReport(IncrementalGoogleAdsStream): """ - DisplayKeywordPerformanceReport stream: https://developers.google.com/google-ads/api/fields/v9/display_keyword_view + DisplayKeywordPerformanceReport stream: https://developers.google.com/google-ads/api/fields/v11/display_keyword_view Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#display_keyword_performance """ class DisplayTopicsPerformanceReport(IncrementalGoogleAdsStream): """ - DisplayTopicsPerformanceReport stream: https://developers.google.com/google-ads/api/fields/v9/topic_view + DisplayTopicsPerformanceReport stream: https://developers.google.com/google-ads/api/fields/v11/topic_view Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#display_topics_performance """ class ShoppingPerformanceReport(IncrementalGoogleAdsStream): """ - ShoppingPerformanceReport stream: https://developers.google.com/google-ads/api/fields/v9/shopping_performance_view + ShoppingPerformanceReport stream: https://developers.google.com/google-ads/api/fields/v11/shopping_performance_view Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#shopping_performance """ class UserLocationReport(IncrementalGoogleAdsStream): """ - UserLocationReport stream: https://developers.google.com/google-ads/api/fields/v9/user_location_view + UserLocationReport stream: https://developers.google.com/google-ads/api/fields/v11/user_location_view Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#geo_performance """ class GeographicReport(IncrementalGoogleAdsStream): """ - UserLocationReport stream: https://developers.google.com/google-ads/api/fields/v9/geographic_view + UserLocationReport stream: https://developers.google.com/google-ads/api/fields/v11/geographic_view """ class KeywordReport(IncrementalGoogleAdsStream): """ - UserLocationReport stream: https://developers.google.com/google-ads/api/fields/v9/keyword_view + UserLocationReport stream: https://developers.google.com/google-ads/api/fields/v11/keyword_view """ class ClickView(IncrementalGoogleAdsStream): """ - ClickView stream: https://developers.google.com/google-ads/api/reference/rpc/v9/ClickView + ClickView stream: https://developers.google.com/google-ads/api/reference/rpc/v11/ClickView """ primary_key = ["click_view.gclid", "segments.date", "segments.ad_network_type"] diff --git a/docs/integrations/sources/google-ads.md b/docs/integrations/sources/google-ads.md index 19df326836e0..ea5181df5fb9 100644 --- a/docs/integrations/sources/google-ads.md +++ b/docs/integrations/sources/google-ads.md @@ -140,6 +140,7 @@ This source is constrained by whatever API limits are set for the Google Ads tha | Version | Date | Pull Request | Subject | |:---------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------| +| `0.1.43` | 2022-07-12 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Update API version to `v11`, update `google-ads` to 17.0.0 | | `0.1.42` | 2022-06-08 | [13624](https://github.com/airbytehq/airbyte/pull/13624) | Update `google-ads` to 15.1.1, pin `protobuf==3.20.0` to work on MacOS M1 machines (AMD) | | `0.1.41` | 2022-06-08 | [13618](https://github.com/airbytehq/airbyte/pull/13618) | Add missing dependency | | `0.1.40` | 2022-06-02 | [13423](https://github.com/airbytehq/airbyte/pull/13423) | Fix the missing data [issue](https://github.com/airbytehq/airbyte/issues/12999) | From 07c416a0563caf071934b027187533c19269ee20 Mon Sep 17 00:00:00 2001 From: Serhii Date: Tue, 12 Jul 2022 13:32:35 +0300 Subject: [PATCH 2/7] Updated PR number --- docs/integrations/sources/google-ads.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/google-ads.md b/docs/integrations/sources/google-ads.md index ea5181df5fb9..d16576b36f36 100644 --- a/docs/integrations/sources/google-ads.md +++ b/docs/integrations/sources/google-ads.md @@ -140,7 +140,7 @@ This source is constrained by whatever API limits are set for the Google Ads tha | Version | Date | Pull Request | Subject | |:---------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------| -| `0.1.43` | 2022-07-12 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Update API version to `v11`, update `google-ads` to 17.0.0 | +| `0.1.43` | 2022-07-12 | [14614](https://github.com/airbytehq/airbyte/pull/14614) | Update API version to `v11`, update `google-ads` to 17.0.0 | | `0.1.42` | 2022-06-08 | [13624](https://github.com/airbytehq/airbyte/pull/13624) | Update `google-ads` to 15.1.1, pin `protobuf==3.20.0` to work on MacOS M1 machines (AMD) | | `0.1.41` | 2022-06-08 | [13618](https://github.com/airbytehq/airbyte/pull/13618) | Add missing dependency | | `0.1.40` | 2022-06-02 | [13423](https://github.com/airbytehq/airbyte/pull/13423) | Fix the missing data [issue](https://github.com/airbytehq/airbyte/issues/12999) | From 7e0df7d761fc358296046381346daa0fdd071318 Mon Sep 17 00:00:00 2001 From: Serhii Date: Wed, 13 Jul 2022 19:27:21 +0300 Subject: [PATCH 3/7] Updated after review --- .../source_google_ads/custom_query_stream.py | 2 +- .../source-google-ads/source_google_ads/google_ads.py | 2 +- .../source-google-ads/source_google_ads/spec.json | 2 +- .../source-google-ads/source_google_ads/streams.py | 6 +++--- .../connectors/source-google-ads/unit_tests/common.py | 2 +- .../connectors/source-google-ads/unit_tests/test_source.py | 2 +- .../connectors/source-google-ads/unit_tests/test_streams.py | 4 ++-- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py index ef9d3408f5f0..73a12a8ad1c9 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py @@ -72,7 +72,7 @@ def get_json_schema(self) -> Dict[str, Any]: # Represents protobuf message and could be anything, set custom # attribute "protobuf_message" to convert it to a string (or # array of strings) later. - # https://developers.google.com/google-ads/api/reference/rpc/v9/GoogleAdsFieldDataTypeEnum.GoogleAdsFieldDataType?hl=en#message + # https://developers.google.com/google-ads/api/reference/rpc/v11/GoogleAdsFieldDataTypeEnum.GoogleAdsFieldDataType?hl=en#message if node.is_repeated: output_type = ["array", "null"] else: diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py index 7fb1f36dbca6..74417089069b 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py @@ -8,7 +8,7 @@ import pendulum from google.ads.googleads.client import GoogleAdsClient -from google.ads.googleads.v9.services.types.google_ads_service import GoogleAdsRow, SearchGoogleAdsResponse +from google.ads.googleads.v11.services.types.google_ads_service import GoogleAdsRow, SearchGoogleAdsResponse from proto.marshal.collections import Repeated, RepeatedComposite REPORT_MAPPING = { diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json index 014b0051587e..128a77125100 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json @@ -90,7 +90,7 @@ "query": { "type": "string", "title": "Custom Query", - "description": "A custom defined GAQL query for building the report. Should not contain segments.date expression because it is used by incremental streams. See Google's query builder for more information.", + "description": "A custom defined GAQL query for building the report. Should not contain segments.date expression because it is used by incremental streams. See Google's query builder for more information.", "examples": [ "SELECT segments.ad_destination_type, campaign.advertising_channel_sub_type FROM campaign WHERE campaign.status = 'PAUSED'" ] diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py index c8e8efcc1299..ac0e661a4bae 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py @@ -9,9 +9,9 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import IncrementalMixin, Stream from google.ads.googleads.errors import GoogleAdsException -from google.ads.googleads.v9.errors.types.authorization_error import AuthorizationErrorEnum -from google.ads.googleads.v9.errors.types.request_error import RequestErrorEnum -from google.ads.googleads.v9.services.services.google_ads_service.pagers import SearchPager +from google.ads.googleads.v11.errors.types.authorization_error import AuthorizationErrorEnum +from google.ads.googleads.v11.errors.types.request_error import RequestErrorEnum +from google.ads.googleads.v11.services.services.google_ads_service.pagers import SearchPager from .google_ads import GoogleAds from .models import Customer diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/common.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/common.py index 0038abf2f2dc..66824c90d6f9 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/common.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/common.py @@ -5,7 +5,7 @@ import json from google.ads.googleads.errors import GoogleAdsException -from google.ads.googleads.v9 import GoogleAdsFailure +from google.ads.googleads.v11 import GoogleAdsFailure class MockSearchRequest: diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py index 961d652e80dd..072e2a0626a4 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py @@ -9,7 +9,7 @@ from airbyte_cdk import AirbyteLogger from freezegun import freeze_time from google.ads.googleads.errors import GoogleAdsException -from google.ads.googleads.v9.errors.types.authorization_error import AuthorizationErrorEnum +from google.ads.googleads.v11.errors.types.authorization_error import AuthorizationErrorEnum from pendulum import today from source_google_ads.custom_query_stream import CustomQuery from source_google_ads.google_ads import GoogleAds diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py index 5973871bc234..d20ac79a62a2 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py @@ -7,8 +7,8 @@ import pytest from airbyte_cdk.models import SyncMode from google.ads.googleads.errors import GoogleAdsException -from google.ads.googleads.v9.errors.types.errors import ErrorCode, GoogleAdsError, GoogleAdsFailure -from google.ads.googleads.v9.errors.types.request_error import RequestErrorEnum +from google.ads.googleads.v11.errors.types.errors import ErrorCode, GoogleAdsError, GoogleAdsFailure +from google.ads.googleads.v11.errors.types.request_error import RequestErrorEnum from grpc import RpcError from source_google_ads.google_ads import GoogleAds from source_google_ads.streams import ClickView From 99e025beb13b8c10c65c44c266eab8b8b3a5cd6c Mon Sep 17 00:00:00 2001 From: Serhii Date: Mon, 18 Jul 2022 20:41:08 +0300 Subject: [PATCH 4/7] Added validation error --- .../source-paypal-transaction/Dockerfile | 2 +- .../source-paypal-transaction/setup.py | 1 + .../source_paypal_transaction/schemas/TODO.md | 25 --- .../source_paypal_transaction/source.py | 78 +++++++- .../source_paypal_transaction/utils.py | 24 +++ .../unit_tests/transaction.json | 171 ++++++++++++++++++ .../unit_tests/unit_test.py | 84 +++++++-- .../sources/paypal-transaction.md | 3 +- 8 files changed, 334 insertions(+), 54 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/TODO.md create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/utils.py create mode 100644 airbyte-integrations/connectors/source-paypal-transaction/unit_tests/transaction.json diff --git a/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile b/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile index 9b4e721e9204..85a3cff58163 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile +++ b/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.1.7 LABEL io.airbyte.name=airbyte/source-paypal-transaction diff --git a/airbyte-integrations/connectors/source-paypal-transaction/setup.py b/airbyte-integrations/connectors/source-paypal-transaction/setup.py index a6daf3f0f24f..8babc8c122b5 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/setup.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/setup.py @@ -12,6 +12,7 @@ TEST_REQUIREMENTS = [ "pytest~=6.1", "pytest-mock~=3.6", + "requests-mock", "source-acceptance-test", ] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/TODO.md b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/TODO.md deleted file mode 100644 index cf1efadb3c9c..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/TODO.md +++ /dev/null @@ -1,25 +0,0 @@ -# TODO: Define your stream schemas -Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). - -The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. - -The schema of a stream is the return value of `Stream.get_json_schema`. - -## Static schemas -By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. - -Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. - -## Dynamic schemas -If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). - -## Dynamically modifying static schemas -Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: -``` -def get_json_schema(self): - schema = super().get_json_schema() - schema['dynamically_determined_property'] = "property" - return schema -``` - -Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 14005c7358dd..cd56f90678a4 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -10,6 +10,7 @@ from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union, Dict import requests +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream @@ -17,6 +18,8 @@ from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator, Oauth2Authenticator from dateutil.parser import isoparse +from .utils import middle_date_slices + class PaypalHttpException(Exception): """HTTPError Exception with detailed info""" @@ -28,16 +31,20 @@ def __str__(self): message = repr(self.error) if self.error.response.content: - content = self.error.response.content.decode() - try: - details = json.loads(content) - except json.decoder.JSONDecodeError: - details = content - + details = self.error_message() message = f"{message} Details: {details}" return message + def error_message(self): + content = self.error.response.content.decode() + try: + details = json.loads(content) + except json.decoder.JSONDecodeError: + details = content + + return details + def __repr__(self): return self.__str__() @@ -72,7 +79,7 @@ class PaypalTransactionStream(HttpStream, ABC): # API limit: (now() - start_date_min) <= start_date <= end_date <= last_refreshed_datetime <= now start_date_min: Mapping[str, int] = {"days": 3 * 365} # API limit - 3 years last_refreshed_datetime: Optional[datetime] = None # extracted from API response. Indicate the most resent possible start_date - stream_slice_period: Mapping[str, int] = {"days": 1} # max period is 31 days (API limit) + stream_slice_period: Mapping[str, int] = {"days": 15} # max period is 31 days (API limit) requests_per_minute: int = 30 # API limit is 50 reqs/min from 1 IP to all endpoints, otherwise IP is banned for 5 mins @@ -179,6 +186,11 @@ def get_field(record: Mapping[str, Any], field_path: Union[List[str], str]): return data + @staticmethod + def max_records_in_response_reached(exception: Exception, **kwargs): + message = exception.error_message() + return message.get("name") == "RESULTSET_TOO_LARGE" + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. @@ -253,6 +265,58 @@ def stream_slices( return slices + def _prepared_request( + self, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + next_page_token: Optional[dict] = None + ): + request_headers = self.request_headers(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + request = self._create_prepared_request( + path=self.path(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + headers=dict(request_headers, **self.authenticator.get_auth_header()), + params=self.request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + json=self.request_body_json(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + data=self.request_body_data(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + ) + request_kwargs = self.request_kwargs(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + + return request, request_kwargs + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Mapping[str, Any]]: + stream_state = stream_state or {} + pagination_complete = False + next_page_token = None + while not pagination_complete: + request, request_kwargs = self._prepared_request(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + + try: + response = self._send_request(request, request_kwargs) + except PaypalHttpException as exception: + if self.max_records_in_response_reached(exception): + date_slices = middle_date_slices(stream_slice) + if date_slices: + for date_slice in date_slices: + yield from self.read_records(sync_mode, cursor_field=cursor_field, stream_slice=date_slice, stream_state=stream_state) + break + else: + raise exception + + yield from self.parse_response(response, stream_state=stream_state, stream_slice=stream_slice) + + next_page_token = self.next_page_token(response) + if not next_page_token: + pagination_complete = True + + # Always return an empty generator just in case no records were ever yielded + yield from [] + def _send_request(self, request: requests.PreparedRequest, request_kwargs: Mapping[str, Any]) -> requests.Response: try: return super()._send_request(request, request_kwargs) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/utils.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/utils.py new file mode 100644 index 000000000000..610623cfe1f6 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/utils.py @@ -0,0 +1,24 @@ +from datetime import datetime + + +def to_datetime_str(date: datetime) -> datetime: + """ + Returns the formated datetime string. + :: Output example: '2021-07-15T0:0:0+00:00' FORMAT : "%Y-%m-%dT%H:%M:%S%z" + """ + return datetime.strptime(date, "%Y-%m-%dT%H:%M:%S%z") + + +def middle_date_slices(stream_slice): + """Returns the mid-split datetime slices.""" + start_date, end_date = to_datetime_str(stream_slice["start_date"]), to_datetime_str(stream_slice["end_date"]) + if start_date < end_date: + middle_date = start_date + (end_date - start_date) / 2 + return [{ + "start_date": start_date.isoformat(), + "end_date": middle_date.isoformat(), + }, + { + "start_date": middle_date.isoformat(), + "end_date": end_date.isoformat(), + }] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/transaction.json b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/transaction.json new file mode 100644 index 000000000000..6823cfb4e77e --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/transaction.json @@ -0,0 +1,171 @@ +{ + "transaction_details": [ + { + "transaction_info": + { + "paypal_account_id": "6STWC2LSUYYYE", + "transaction_id": "5TY05013RG002845M", + "transaction_event_code": "T0006", + "transaction_initiation_date": "2014-07-11T04:03:52+0000", + "transaction_updated_date": "2014-07-11T04:03:52+0000", + "transaction_amount": + { + "currency_code": "USD", + "value": "465.00" + }, + "fee_amount": + { + "currency_code": "USD", + "value": "-13.79" + }, + "insurance_amount": + { + "currency_code": "USD", + "value": "15.00" + }, + "shipping_amount": + { + "currency_code": "USD", + "value": "30.00" + }, + "shipping_discount_amount": + { + "currency_code": "USD", + "value": "10.00" + }, + "transaction_status": "S", + "transaction_subject": "Bill for your purchase", + "transaction_note": "Check out the latest sales", + "invoice_id": "Invoice-005", + "custom_field": "Thank you for your business", + "protection_eligibility": "01" + }, + "payer_info": + { + "account_id": "6STWC2LSUYYYE", + "email_address": "consumer@example.com", + "address_status": "Y", + "payer_status": "Y", + "payer_name": + { + "given_name": "test", + "surname": "consumer", + "alternate_full_name": "test consumer" + }, + "country_code": "US" + }, + "shipping_info": + { + "name": "Sowmith", + "address": + { + "line1": "Eco Space, bellandur", + "line2": "OuterRingRoad", + "city": "Bangalore", + "country_code": "IN", + "postal_code": "560103" + } + }, + "cart_info": + { + "item_details": [ + { + "item_code": "ItemCode-1", + "item_name": "Item1 - radio", + "item_description": "Radio", + "item_quantity": "2", + "item_unit_price": + { + "currency_code": "USD", + "value": "50.00" + }, + "item_amount": + { + "currency_code": "USD", + "value": "100.00" + }, + "tax_amounts": [ + { + "tax_amount": + { + "currency_code": "USD", + "value": "20.00" + } + }], + "total_item_amount": + { + "currency_code": "USD", + "value": "120.00" + }, + "invoice_number": "Invoice-005" + }, + { + "item_code": "ItemCode-2", + "item_name": "Item2 - Headset", + "item_description": "Headset", + "item_quantity": "3", + "item_unit_price": + { + "currency_code": "USD", + "value": "100.00" + }, + "item_amount": + { + "currency_code": "USD", + "value": "300.00" + }, + "tax_amounts": [ + { + "tax_amount": + { + "currency_code": "USD", + "value": "60.00" + } + }], + "total_item_amount": + { + "currency_code": "USD", + "value": "360.00" + }, + "invoice_number": "Invoice-005" + }, + { + "item_name": "3", + "item_quantity": "1", + "item_unit_price": + { + "currency_code": "USD", + "value": "-50.00" + }, + "item_amount": + { + "currency_code": "USD", + "value": "-50.00" + }, + "total_item_amount": + { + "currency_code": "USD", + "value": "-50.00" + }, + "invoice_number": "Invoice-005" + }] + }, + "store_info": + {}, + "auction_info": + {}, + "incentive_info": + {} + }], + "account_number": "XZXSPECPDZHZU", + "last_refreshed_datetime": "2017-01-02T06:59:59+0000", + "page": 1, + "total_items": 1, + "total_pages": 1, + "links": [ + { + "href": "https://api-m.sandbox.paypal.com/v1/reporting/transactions?transaction_id=5TY05013RG002845M&fields=all&page_size=100&page=1", + "rel": "self", + "method": "GET" + }] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py index 0b2b455389bf..48ebb2565a5d 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py @@ -1,12 +1,13 @@ # # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # - +import json +import pathlib from datetime import datetime, timedelta from airbyte_cdk.sources.streams.http.auth import NoAuth from dateutil.parser import isoparse -from pytest import fixture +from pytest import fixture, raises from source_paypal_transaction.source import Balances, PaypalTransactionStream, Transactions @@ -16,8 +17,15 @@ def time_sleep_mock(mocker): yield time_mock -def test_get_field(): +@fixture(autouse=True) +def transactions(request): + file = pathlib.Path(request.node.fspath.strpath) + transaction = file.with_name('transaction.json') + with transaction.open() as fp: + return json.load(fp) + +def test_get_field(): record = {"a": {"b": {"c": "d"}}} # Test expected result - field_path is a list assert "d" == PaypalTransactionStream.get_field(record, field_path=["a", "b", "c"]) @@ -67,7 +75,6 @@ def now(): def test_transactions_stream_slices(): - start_date_max = {"hours": 0} # if start_date > now - **start_date_max then no slices @@ -109,6 +116,7 @@ def test_transactions_stream_slices(): start_date=now() - timedelta(**start_date_max) - timedelta(days=1), ) transactions.get_last_refreshed_datetime = lambda x: None + transactions.stream_slice_period = {"days": 1} stream_slices = transactions.stream_slices(sync_mode="any") assert 2 == len(stream_slices) @@ -117,6 +125,7 @@ def test_transactions_stream_slices(): start_date=now() - timedelta(**start_date_max) - timedelta(days=1, hours=2), ) transactions.get_last_refreshed_datetime = lambda x: None + transactions.stream_slice_period = {"days": 1} stream_slices = transactions.stream_slices(sync_mode="any") assert 2 == len(stream_slices) @@ -125,6 +134,7 @@ def test_transactions_stream_slices(): start_date=now() - timedelta(**start_date_max) - timedelta(days=30, minutes=1), ) transactions.get_last_refreshed_datetime = lambda x: None + transactions.stream_slice_period = {"days": 1} stream_slices = transactions.stream_slices(sync_mode="any") assert 31 == len(stream_slices) @@ -135,13 +145,14 @@ def test_transactions_stream_slices(): end_date=isoparse("2021-06-04T12:00:00+00:00"), ) transactions.get_last_refreshed_datetime = lambda x: None + transactions.stream_slice_period = {"days": 1} stream_slices = transactions.stream_slices(sync_mode="any") assert [ - {"start_date": "2021-06-01T10:00:00+00:00", "end_date": "2021-06-02T10:00:00+00:00"}, - {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, - {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-04T10:00:00+00:00"}, - {"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}, - ] == stream_slices + {"start_date": "2021-06-01T10:00:00+00:00", "end_date": "2021-06-02T10:00:00+00:00"}, + {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, + {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-04T10:00:00+00:00"}, + {"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}, + ] == stream_slices # tests with specified end_date and stream_state transactions = Transactions( @@ -150,12 +161,13 @@ def test_transactions_stream_slices(): end_date=isoparse("2021-06-04T12:00:00+00:00"), ) transactions.get_last_refreshed_datetime = lambda x: None + transactions.stream_slice_period = {"days": 1} stream_slices = transactions.stream_slices(sync_mode="any", stream_state={"date": "2021-06-02T10:00:00+00:00"}) assert [ - {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, - {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-04T10:00:00+00:00"}, - {"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}, - ] == stream_slices + {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, + {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-04T10:00:00+00:00"}, + {"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}, + ] == stream_slices transactions = Transactions( authenticator=NoAuth(), @@ -197,6 +209,7 @@ def test_balances_stream_slices(): start_date=now - timedelta(days=1), ) balance.get_last_refreshed_datetime = lambda x: None + balance.stream_slice_period = {"days": 1} stream_slices = balance.stream_slices(sync_mode="any") assert 2 == len(stream_slices) @@ -205,6 +218,7 @@ def test_balances_stream_slices(): start_date=now - timedelta(days=1, minutes=1), ) balance.get_last_refreshed_datetime = lambda x: None + balance.stream_slice_period = {"days": 1} stream_slices = balance.stream_slices(sync_mode="any") assert 2 == len(stream_slices) @@ -215,12 +229,13 @@ def test_balances_stream_slices(): end_date=isoparse("2021-06-03T12:00:00+00:00"), ) balance.get_last_refreshed_datetime = lambda x: None + balance.stream_slice_period = {"days": 1} stream_slices = balance.stream_slices(sync_mode="any") assert [ - {"start_date": "2021-06-01T10:00:00+00:00", "end_date": "2021-06-02T10:00:00+00:00"}, - {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, - {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}, - ] == stream_slices + {"start_date": "2021-06-01T10:00:00+00:00", "end_date": "2021-06-02T10:00:00+00:00"}, + {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, + {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}, + ] == stream_slices # Test with stream state balance = Balances( @@ -229,11 +244,12 @@ def test_balances_stream_slices(): end_date=isoparse("2021-06-03T12:00:00+00:00"), ) balance.get_last_refreshed_datetime = lambda x: None + balance.stream_slice_period = {"days": 1} stream_slices = balance.stream_slices(sync_mode="any", stream_state={"date": "2021-06-02T10:00:00+00:00"}) assert [ - {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, - {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}, - ] == stream_slices + {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, + {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}, + ] == stream_slices balance = Balances( authenticator=NoAuth(), @@ -241,6 +257,7 @@ def test_balances_stream_slices(): end_date=isoparse("2021-06-03T12:00:00+00:00"), ) balance.get_last_refreshed_datetime = lambda x: None + balance.stream_slice_period = {"days": 1} stream_slices = balance.stream_slices(sync_mode="any", stream_state={"date": "2021-06-03T11:00:00+00:00"}) assert [{"start_date": "2021-06-03T11:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}] == stream_slices @@ -250,5 +267,32 @@ def test_balances_stream_slices(): end_date=isoparse("2021-06-03T12:00:00+00:00"), ) balance.get_last_refreshed_datetime = lambda x: None + balance.stream_slice_period = {"days": 1} stream_slices = balance.stream_slices(sync_mode="any", stream_state={"date": "2021-06-03T12:00:00+00:00"}) assert [{"start_date": "2021-06-03T12:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}] == stream_slices + + +def test_max_records_in_response_reached(transactions, requests_mock): + balance = Transactions( + authenticator=NoAuth(), + start_date=isoparse("2021-07-01T10:00:00+00:00"), + end_date=isoparse("2021-07-29T12:00:00+00:00"), + ) + error_message = {"name": "RESULTSET_TOO_LARGE", "message": "Result set size is greater than the maximum limit. Change the filter " + "criteria and try again."} + url = "https://api-m.paypal.com/v1/reporting/transactions" + + requests_mock.register_uri('GET', url + "?start_date=2021-07-01T12%3A00%3A00%2B00%3A00&end_date=2021-07-29T12%3A00%3A00%2B00%3A00", + json=error_message, status_code=400) + requests_mock.register_uri('GET', url + "?start_date=2021-07-01T12%3A00%3A00%2B00%3A00&end_date=2021-07-15T12%3A00%3A00%2B00%3A00", + json=transactions) + requests_mock.register_uri('GET', url + "?start_date=2021-07-15T12%3A00%3A00%2B00%3A00&end_date=2021-07-29T12%3A00%3A00%2B00%3A00", + json=transactions) + month_date_slice = {"start_date": "2021-07-01T12:00:00+00:00", "end_date": "2021-07-29T12:00:00+00:00"} + assert len(list(balance.read_records(sync_mode="any", stream_slice=month_date_slice))) == 2 + + requests_mock.register_uri('GET', url + "?start_date=2021-07-01T12%3A00%3A00%2B00%3A00&end_date=2021-07-01T12%3A00%3A00%2B00%3A00", + json=error_message, status_code=400) + one_day_slice = {"start_date": "2021-07-01T12:00:00+00:00", "end_date": "2021-07-01T12:00:00+00:00"} + with raises(Exception): + assert next(balance.read_records(sync_mode="any", stream_slice=one_day_slice)) diff --git a/docs/integrations/sources/paypal-transaction.md b/docs/integrations/sources/paypal-transaction.md index 2abb9e31a915..f915d1ea824c 100644 --- a/docs/integrations/sources/paypal-transaction.md +++ b/docs/integrations/sources/paypal-transaction.md @@ -57,11 +57,12 @@ Transactions sync is performed with default `stream_slice_period` = 1 day, it me | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------| +| 0.1.7 | 2022-07-18 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Adding `RESULTSET_TOO_LARGE` error validation | | 0.1.6 | 2022-06-10 | [13682](https://github.com/airbytehq/airbyte/pull/13682) | Update paypal transaction schema | | 0.1.5 | 2022-04-27 | [12335](https://github.com/airbytehq/airbyte/pull/12335) | Adding fixtures to mock time.sleep for connectors that explicitly sleep | | 0.1.4 | 2021-12-22 | [9034](https://github.com/airbytehq/airbyte/pull/9034) | Update connector fields title/description | | 0.1.3 | 2021-12-16 | [8580](https://github.com/airbytehq/airbyte/pull/8580) | Added more logs during `check connection` stage | | 0.1.2 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | -| 0.1.1 | 2021-08-03 | [5155](https://github.com/airbytehq/airbyte/pull/5155) | fix start\_date\_min limit | +| 0.1.1 | 2021-08-03 | [5155](https://github.com/airbytehq/airbyte/pull/5155) | Fix start\_date\_min limit | | 0.1.0 | 2021-06-10 | [4240](https://github.com/airbytehq/airbyte/pull/4240) | PayPal Transaction Search API | From 6309b10e352b6389ca24e19faa8fc08380f648e5 Mon Sep 17 00:00:00 2001 From: Serhii Date: Tue, 19 Jul 2022 21:33:07 +0300 Subject: [PATCH 5/7] Fixed to linter --- .../source_paypal_transaction/source.py | 19 +- .../source_paypal_transaction/utils.py | 16 +- .../unit_tests/transaction.json | 278 ++++++++---------- .../unit_tests/unit_test.py | 67 +++-- 4 files changed, 190 insertions(+), 190 deletions(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index cd56f90678a4..bfd5724e1067 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -7,15 +7,15 @@ import time from abc import ABC from datetime import datetime, timedelta -from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union, Dict +from typing import Any, Callable, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator, Oauth2Authenticator +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from dateutil.parser import isoparse from .utils import middle_date_slices @@ -266,10 +266,7 @@ def stream_slices( return slices def _prepared_request( - self, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - next_page_token: Optional[dict] = None + self, stream_slice: Mapping[str, Any] = None, stream_state: Mapping[str, Any] = None, next_page_token: Optional[dict] = None ): request_headers = self.request_headers(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) request = self._create_prepared_request( @@ -294,7 +291,9 @@ def read_records( pagination_complete = False next_page_token = None while not pagination_complete: - request, request_kwargs = self._prepared_request(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + request, request_kwargs = self._prepared_request( + stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token + ) try: response = self._send_request(request, request_kwargs) @@ -303,7 +302,9 @@ def read_records( date_slices = middle_date_slices(stream_slice) if date_slices: for date_slice in date_slices: - yield from self.read_records(sync_mode, cursor_field=cursor_field, stream_slice=date_slice, stream_state=stream_state) + yield from self.read_records( + sync_mode, cursor_field=cursor_field, stream_slice=date_slice, stream_state=stream_state + ) break else: raise exception @@ -365,7 +366,7 @@ def request_params( "page_size": self.page_size, "page": page_number, } - + @transformer.registerCustomTransform def transform_function(original_value: Any, field_schema: Dict[str, Any]) -> Any: if isinstance(original_value, str) and field_schema["type"] == "number": diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/utils.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/utils.py index 610623cfe1f6..ad6c0232dce4 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/utils.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/utils.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + from datetime import datetime @@ -14,11 +18,13 @@ def middle_date_slices(stream_slice): start_date, end_date = to_datetime_str(stream_slice["start_date"]), to_datetime_str(stream_slice["end_date"]) if start_date < end_date: middle_date = start_date + (end_date - start_date) / 2 - return [{ - "start_date": start_date.isoformat(), - "end_date": middle_date.isoformat(), - }, + return [ + { + "start_date": start_date.isoformat(), + "end_date": middle_date.isoformat(), + }, { "start_date": middle_date.isoformat(), "end_date": end_date.isoformat(), - }] + }, + ] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/transaction.json b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/transaction.json index 6823cfb4e77e..00621a91e6d1 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/transaction.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/transaction.json @@ -1,171 +1,151 @@ { "transaction_details": [ - { - "transaction_info": { - "paypal_account_id": "6STWC2LSUYYYE", - "transaction_id": "5TY05013RG002845M", - "transaction_event_code": "T0006", - "transaction_initiation_date": "2014-07-11T04:03:52+0000", - "transaction_updated_date": "2014-07-11T04:03:52+0000", - "transaction_amount": - { - "currency_code": "USD", - "value": "465.00" - }, - "fee_amount": - { - "currency_code": "USD", - "value": "-13.79" - }, - "insurance_amount": - { - "currency_code": "USD", - "value": "15.00" - }, - "shipping_amount": - { - "currency_code": "USD", - "value": "30.00" - }, - "shipping_discount_amount": - { - "currency_code": "USD", - "value": "10.00" - }, - "transaction_status": "S", - "transaction_subject": "Bill for your purchase", - "transaction_note": "Check out the latest sales", - "invoice_id": "Invoice-005", - "custom_field": "Thank you for your business", - "protection_eligibility": "01" - }, - "payer_info": - { - "account_id": "6STWC2LSUYYYE", - "email_address": "consumer@example.com", - "address_status": "Y", - "payer_status": "Y", - "payer_name": - { - "given_name": "test", - "surname": "consumer", - "alternate_full_name": "test consumer" - }, - "country_code": "US" - }, - "shipping_info": - { - "name": "Sowmith", - "address": - { - "line1": "Eco Space, bellandur", - "line2": "OuterRingRoad", - "city": "Bangalore", - "country_code": "IN", - "postal_code": "560103" - } - }, - "cart_info": - { - "item_details": [ - { - "item_code": "ItemCode-1", - "item_name": "Item1 - radio", - "item_description": "Radio", - "item_quantity": "2", - "item_unit_price": - { + "transaction_info": { + "paypal_account_id": "6STWC2LSUYYYE", + "transaction_id": "5TY05013RG002845M", + "transaction_event_code": "T0006", + "transaction_initiation_date": "2014-07-11T04:03:52+0000", + "transaction_updated_date": "2014-07-11T04:03:52+0000", + "transaction_amount": { "currency_code": "USD", - "value": "50.00" + "value": "465.00" }, - "item_amount": - { + "fee_amount": { "currency_code": "USD", - "value": "100.00" + "value": "-13.79" }, - "tax_amounts": [ - { - "tax_amount": - { - "currency_code": "USD", - "value": "20.00" - } - }], - "total_item_amount": - { + "insurance_amount": { "currency_code": "USD", - "value": "120.00" + "value": "15.00" }, - "invoice_number": "Invoice-005" - }, - { - "item_code": "ItemCode-2", - "item_name": "Item2 - Headset", - "item_description": "Headset", - "item_quantity": "3", - "item_unit_price": - { + "shipping_amount": { "currency_code": "USD", - "value": "100.00" + "value": "30.00" }, - "item_amount": - { + "shipping_discount_amount": { "currency_code": "USD", - "value": "300.00" + "value": "10.00" + }, + "transaction_status": "S", + "transaction_subject": "Bill for your purchase", + "transaction_note": "Check out the latest sales", + "invoice_id": "Invoice-005", + "custom_field": "Thank you for your business", + "protection_eligibility": "01" + }, + "payer_info": { + "account_id": "6STWC2LSUYYYE", + "email_address": "consumer@example.com", + "address_status": "Y", + "payer_status": "Y", + "payer_name": { + "given_name": "test", + "surname": "consumer", + "alternate_full_name": "test consumer" }, - "tax_amounts": [ - { - "tax_amount": + "country_code": "US" + }, + "shipping_info": { + "name": "Sowmith", + "address": { + "line1": "Eco Space, bellandur", + "line2": "OuterRingRoad", + "city": "Bangalore", + "country_code": "IN", + "postal_code": "560103" + } + }, + "cart_info": { + "item_details": [ + { + "item_code": "ItemCode-1", + "item_name": "Item1 - radio", + "item_description": "Radio", + "item_quantity": "2", + "item_unit_price": { + "currency_code": "USD", + "value": "50.00" + }, + "item_amount": { + "currency_code": "USD", + "value": "100.00" + }, + "tax_amounts": [ + { + "tax_amount": { + "currency_code": "USD", + "value": "20.00" + } + } + ], + "total_item_amount": { + "currency_code": "USD", + "value": "120.00" + }, + "invoice_number": "Invoice-005" + }, + { + "item_code": "ItemCode-2", + "item_name": "Item2 - Headset", + "item_description": "Headset", + "item_quantity": "3", + "item_unit_price": { + "currency_code": "USD", + "value": "100.00" + }, + "item_amount": { + "currency_code": "USD", + "value": "300.00" + }, + "tax_amounts": [ + { + "tax_amount": { + "currency_code": "USD", + "value": "60.00" + } + } + ], + "total_item_amount": { + "currency_code": "USD", + "value": "360.00" + }, + "invoice_number": "Invoice-005" + }, { - "currency_code": "USD", - "value": "60.00" + "item_name": "3", + "item_quantity": "1", + "item_unit_price": { + "currency_code": "USD", + "value": "-50.00" + }, + "item_amount": { + "currency_code": "USD", + "value": "-50.00" + }, + "total_item_amount": { + "currency_code": "USD", + "value": "-50.00" + }, + "invoice_number": "Invoice-005" } - }], - "total_item_amount": - { - "currency_code": "USD", - "value": "360.00" - }, - "invoice_number": "Invoice-005" + ] }, - { - "item_name": "3", - "item_quantity": "1", - "item_unit_price": - { - "currency_code": "USD", - "value": "-50.00" - }, - "item_amount": - { - "currency_code": "USD", - "value": "-50.00" - }, - "total_item_amount": - { - "currency_code": "USD", - "value": "-50.00" - }, - "invoice_number": "Invoice-005" - }] - }, - "store_info": - {}, - "auction_info": - {}, - "incentive_info": - {} - }], + "store_info": {}, + "auction_info": {}, + "incentive_info": {} + } + ], "account_number": "XZXSPECPDZHZU", "last_refreshed_datetime": "2017-01-02T06:59:59+0000", "page": 1, "total_items": 1, "total_pages": 1, "links": [ - { - "href": "https://api-m.sandbox.paypal.com/v1/reporting/transactions?transaction_id=5TY05013RG002845M&fields=all&page_size=100&page=1", - "rel": "self", - "method": "GET" - }] -} \ No newline at end of file + { + "href": "https://api-m.sandbox.paypal.com/v1/reporting/transactions?transaction_id=5TY05013RG002845M&fields=all&page_size=100&page=1", + "rel": "self", + "method": "GET" + } + ] +} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py index 48ebb2565a5d..a784b1122393 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py @@ -1,6 +1,7 @@ # # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # + import json import pathlib from datetime import datetime, timedelta @@ -20,7 +21,7 @@ def time_sleep_mock(mocker): @fixture(autouse=True) def transactions(request): file = pathlib.Path(request.node.fspath.strpath) - transaction = file.with_name('transaction.json') + transaction = file.with_name("transaction.json") with transaction.open() as fp: return json.load(fp) @@ -148,11 +149,11 @@ def test_transactions_stream_slices(): transactions.stream_slice_period = {"days": 1} stream_slices = transactions.stream_slices(sync_mode="any") assert [ - {"start_date": "2021-06-01T10:00:00+00:00", "end_date": "2021-06-02T10:00:00+00:00"}, - {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, - {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-04T10:00:00+00:00"}, - {"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}, - ] == stream_slices + {"start_date": "2021-06-01T10:00:00+00:00", "end_date": "2021-06-02T10:00:00+00:00"}, + {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, + {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-04T10:00:00+00:00"}, + {"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}, + ] == stream_slices # tests with specified end_date and stream_state transactions = Transactions( @@ -164,10 +165,10 @@ def test_transactions_stream_slices(): transactions.stream_slice_period = {"days": 1} stream_slices = transactions.stream_slices(sync_mode="any", stream_state={"date": "2021-06-02T10:00:00+00:00"}) assert [ - {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, - {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-04T10:00:00+00:00"}, - {"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}, - ] == stream_slices + {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, + {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-04T10:00:00+00:00"}, + {"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}, + ] == stream_slices transactions = Transactions( authenticator=NoAuth(), @@ -232,10 +233,10 @@ def test_balances_stream_slices(): balance.stream_slice_period = {"days": 1} stream_slices = balance.stream_slices(sync_mode="any") assert [ - {"start_date": "2021-06-01T10:00:00+00:00", "end_date": "2021-06-02T10:00:00+00:00"}, - {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, - {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}, - ] == stream_slices + {"start_date": "2021-06-01T10:00:00+00:00", "end_date": "2021-06-02T10:00:00+00:00"}, + {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, + {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}, + ] == stream_slices # Test with stream state balance = Balances( @@ -247,9 +248,9 @@ def test_balances_stream_slices(): balance.stream_slice_period = {"days": 1} stream_slices = balance.stream_slices(sync_mode="any", stream_state={"date": "2021-06-02T10:00:00+00:00"}) assert [ - {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, - {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}, - ] == stream_slices + {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, + {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}, + ] == stream_slices balance = Balances( authenticator=NoAuth(), @@ -278,21 +279,33 @@ def test_max_records_in_response_reached(transactions, requests_mock): start_date=isoparse("2021-07-01T10:00:00+00:00"), end_date=isoparse("2021-07-29T12:00:00+00:00"), ) - error_message = {"name": "RESULTSET_TOO_LARGE", "message": "Result set size is greater than the maximum limit. Change the filter " - "criteria and try again."} + error_message = { + "name": "RESULTSET_TOO_LARGE", + "message": "Result set size is greater than the maximum limit. Change the filter " "criteria and try again.", + } url = "https://api-m.paypal.com/v1/reporting/transactions" - requests_mock.register_uri('GET', url + "?start_date=2021-07-01T12%3A00%3A00%2B00%3A00&end_date=2021-07-29T12%3A00%3A00%2B00%3A00", - json=error_message, status_code=400) - requests_mock.register_uri('GET', url + "?start_date=2021-07-01T12%3A00%3A00%2B00%3A00&end_date=2021-07-15T12%3A00%3A00%2B00%3A00", - json=transactions) - requests_mock.register_uri('GET', url + "?start_date=2021-07-15T12%3A00%3A00%2B00%3A00&end_date=2021-07-29T12%3A00%3A00%2B00%3A00", - json=transactions) + requests_mock.register_uri( + "GET", + url + "?start_date=2021-07-01T12%3A00%3A00%2B00%3A00&end_date=2021-07-29T12%3A00%3A00%2B00%3A00", + json=error_message, + status_code=400, + ) + requests_mock.register_uri( + "GET", url + "?start_date=2021-07-01T12%3A00%3A00%2B00%3A00&end_date=2021-07-15T12%3A00%3A00%2B00%3A00", json=transactions + ) + requests_mock.register_uri( + "GET", url + "?start_date=2021-07-15T12%3A00%3A00%2B00%3A00&end_date=2021-07-29T12%3A00%3A00%2B00%3A00", json=transactions + ) month_date_slice = {"start_date": "2021-07-01T12:00:00+00:00", "end_date": "2021-07-29T12:00:00+00:00"} assert len(list(balance.read_records(sync_mode="any", stream_slice=month_date_slice))) == 2 - requests_mock.register_uri('GET', url + "?start_date=2021-07-01T12%3A00%3A00%2B00%3A00&end_date=2021-07-01T12%3A00%3A00%2B00%3A00", - json=error_message, status_code=400) + requests_mock.register_uri( + "GET", + url + "?start_date=2021-07-01T12%3A00%3A00%2B00%3A00&end_date=2021-07-01T12%3A00%3A00%2B00%3A00", + json=error_message, + status_code=400, + ) one_day_slice = {"start_date": "2021-07-01T12:00:00+00:00", "end_date": "2021-07-01T12:00:00+00:00"} with raises(Exception): assert next(balance.read_records(sync_mode="any", stream_slice=one_day_slice)) From 70489355b2136a5c1dabc2ee743bf811e777703c Mon Sep 17 00:00:00 2001 From: Serhii Date: Tue, 19 Jul 2022 21:36:56 +0300 Subject: [PATCH 6/7] Updated PR number --- docs/integrations/sources/paypal-transaction.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/paypal-transaction.md b/docs/integrations/sources/paypal-transaction.md index f915d1ea824c..32c0b71fe6a8 100644 --- a/docs/integrations/sources/paypal-transaction.md +++ b/docs/integrations/sources/paypal-transaction.md @@ -57,7 +57,7 @@ Transactions sync is performed with default `stream_slice_period` = 1 day, it me | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------| -| 0.1.7 | 2022-07-18 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Adding `RESULTSET_TOO_LARGE` error validation | +| 0.1.7 | 2022-07-18 | [14804](https://github.com/airbytehq/airbyte/pull/14804) | Adding `RESULTSET_TOO_LARGE` error validation | | 0.1.6 | 2022-06-10 | [13682](https://github.com/airbytehq/airbyte/pull/13682) | Update paypal transaction schema | | 0.1.5 | 2022-04-27 | [12335](https://github.com/airbytehq/airbyte/pull/12335) | Adding fixtures to mock time.sleep for connectors that explicitly sleep | | 0.1.4 | 2021-12-22 | [9034](https://github.com/airbytehq/airbyte/pull/9034) | Update connector fields title/description | From 2ad2f79b2dfded2c1b94cc0bea3106d1617743c1 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Tue, 19 Jul 2022 18:51:58 +0000 Subject: [PATCH 7/7] auto-bump connector version [ci skip] --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 5462461d8ab2..b1e76eec2589 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -683,7 +683,7 @@ - name: Paypal Transaction sourceDefinitionId: d913b0f2-cc51-4e55-a44c-8ba1697b9239 dockerRepository: airbyte/source-paypal-transaction - dockerImageTag: 0.1.6 + dockerImageTag: 0.1.7 documentationUrl: https://docs.airbyte.io/integrations/sources/paypal-transaction icon: paypal.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index b4db3d4ec3c2..2c23290c01ba 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -6714,7 +6714,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-paypal-transaction:0.1.6" +- dockerImage: "airbyte/source-paypal-transaction:0.1.7" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/paypal-transactions" connectionSpecification: