diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e094cb9a-26de-4645-8761-65c0c425d1de.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e094cb9a-26de-4645-8761-65c0c425d1de.json index b2a853de5919..712f4c4558c8 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e094cb9a-26de-4645-8761-65c0c425d1de.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e094cb9a-26de-4645-8761-65c0c425d1de.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "e094cb9a-26de-4645-8761-65c0c425d1de", "name": "Stripe", "dockerRepository": "airbyte/source-stripe", - "dockerImageTag": "0.1.20", + "dockerImageTag": "0.1.21", "documentationUrl": "https://docs.airbyte.io/integrations/sources/stripe", "icon": "stripe.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index a73ae9aa4316..5091fc776e42 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -126,7 +126,7 @@ - sourceDefinitionId: e094cb9a-26de-4645-8761-65c0c425d1de name: Stripe dockerRepository: airbyte/source-stripe - dockerImageTag: 0.1.20 + dockerImageTag: 0.1.21 documentationUrl: https://docs.airbyte.io/integrations/sources/stripe icon: stripe.svg sourceType: api diff --git a/airbyte-integrations/connectors/source-stripe/Dockerfile b/airbyte-integrations/connectors/source-stripe/Dockerfile index 6c6b8e6a1400..b6467d211d35 100644 --- a/airbyte-integrations/connectors/source-stripe/Dockerfile +++ b/airbyte-integrations/connectors/source-stripe/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.20 +LABEL io.airbyte.version=0.1.21 LABEL io.airbyte.name=airbyte/source-stripe diff --git a/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml b/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml index 5e7808e11384..3f308f6e0d85 100644 --- a/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml @@ -13,24 +13,28 @@ tests: - config_path: "secrets/config.json" - config_path: "secrets/connected_account_config.json" basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/full_refresh_configured_catalog.json" # TEST 1 - Reading catalog without invoice_line_items - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/non_invoice_line_items_catalog.json" timeout_seconds: 3600 - # TEST 2 - Reading data from account that has no records for stream Disputes +# # TEST 2 - Reading data from account that has no records for stream Disputes - config_path: "secrets/connected_account_config.json" - configured_catalog_path: "integration_tests/non_disputes_events_catalog.json" + configured_catalog_path: "integration_tests/connected_account_configured_catalog.json" timeout_seconds: 3600 incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/non_invoice_line_items_catalog.json" future_state_path: "integration_tests/abnormal_state.json" - config_path: "secrets/connected_account_config.json" - configured_catalog_path: "integration_tests/non_disputes_events_catalog.json" + configured_catalog_path: "integration_tests/connected_account_configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/non_invoice_line_items_catalog.json" timeout_seconds: 3600 + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/full_refresh_configured_catalog.json" - config_path: "secrets/connected_account_config.json" - + configured_catalog_path: "integration_tests/connected_account_configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-stripe/integration_tests/configured_catalog.json index 07f2b651b958..bbe8a73f9fb2 100644 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-stripe/integration_tests/configured_catalog.json @@ -1,5 +1,90 @@ { "streams": [ + { + "stream": { + "name": "balance_transactions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["created"] + }, + { + "stream": { + "name": "bank_accounts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "charges", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["created"] + }, + { + "stream": { + "name": "coupons", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["created"] + }, + { + "stream": { + "name": "customer_balance_transactions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "customers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["created"] + }, + { + "stream": { + "name": "disputes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["created"] + }, { "stream": { "name": "events", @@ -13,16 +98,142 @@ "destination_sync_mode": "overwrite", "cursor_field": ["created"] }, + { + "stream": { + "name": "invoice_items", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["date"] + }, + { + "stream": { + "name": "invoice_line_items", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "invoices", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["created"] + }, { "stream": { "name": "payment_intents", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created"] + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["created"] + }, + { + "stream": { + "name": "payouts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["created"] + }, + { + "stream": { + "name": "plans", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "overwrite", + "cursor_field": ["created"] + }, + { + "stream": { + "name": "products", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["created"] + }, + { + "stream": { + "name": "refunds", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["created"] + }, + { + "stream": { + "name": "subscription_items", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "subscriptions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["created"] + }, + { + "stream": { + "name": "transfers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["created"] } ] } diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/non_disputes_events_catalog.json b/airbyte-integrations/connectors/source-stripe/integration_tests/connected_account_configured_catalog.json similarity index 100% rename from airbyte-integrations/connectors/source-stripe/integration_tests/non_disputes_events_catalog.json rename to airbyte-integrations/connectors/source-stripe/integration_tests/connected_account_configured_catalog.json diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/full_refresh_configured_catalog.json b/airbyte-integrations/connectors/source-stripe/integration_tests/full_refresh_configured_catalog.json new file mode 100644 index 000000000000..e820bbce21c9 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/integration_tests/full_refresh_configured_catalog.json @@ -0,0 +1,23 @@ +{ + "streams": [ + { + "stream": { + "name": "bank_accounts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "customer_balance_transactions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/non_invoice_line_items_catalog.json b/airbyte-integrations/connectors/source-stripe/integration_tests/non_invoice_line_items_catalog.json index 95f780f70c95..020378c92450 100644 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/non_invoice_line_items_catalog.json +++ b/airbyte-integrations/connectors/source-stripe/integration_tests/non_invoice_line_items_catalog.json @@ -2,7 +2,7 @@ "streams": [ { "stream": { - "name": "subscriptions", + "name": "customers", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -15,7 +15,7 @@ }, { "stream": { - "name": "customers", + "name": "disputes", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -28,7 +28,111 @@ }, { "stream": { - "name": "disputes", + "name": "balance_transactions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["created"] + }, + { + "stream": { + "name": "charges", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["created"] + }, + { + "stream": { + "name": "coupons", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["created"] + }, + { + "stream": { + "name": "invoice_items", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["date"] + }, + { + "stream": { + "name": "payouts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["created"] + }, + { + "stream": { + "name": "plans", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["created"] + }, + { + "stream": { + "name": "refunds", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["created"] + }, + { + "stream": { + "name": "transfers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["created"] + }, + { + "stream": { + "name": "events", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/subscription_catalog.json b/airbyte-integrations/connectors/source-stripe/integration_tests/subscription_catalog.json deleted file mode 100644 index 3bbe1c35bb55..000000000000 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/subscription_catalog.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "subscriptions", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"] - } - ] -} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py index fd216cfe6220..3063b3ae52d5 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py @@ -5,6 +5,7 @@ from typing import Any, List, Mapping, Tuple +import pendulum import stripe from airbyte_cdk import AirbyteLogger from airbyte_cdk.sources import AbstractSource @@ -44,8 +45,9 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = TokenAuthenticator(config["client_secret"]) - args = {"authenticator": authenticator, "account_id": config["account_id"]} - incremental_args = {**args, "lookback_window_days": config.get("lookback_window_days"), "start_date": config["start_date"]} + start_date = pendulum.parse(config["start_date"]).int_timestamp + args = {"authenticator": authenticator, "account_id": config["account_id"], "start_date": start_date} + incremental_args = {**args, "lookback_window_days": config.get("lookback_window_days")} return [ BalanceTransactions(**incremental_args), BankAccounts(**args), diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py index af9ece25f1a9..472c8533ef5d 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py @@ -17,9 +17,10 @@ class StripeStream(HttpStream, ABC): url_base = "https://api.stripe.com/v1/" primary_key = "id" - def __init__(self, account_id: str, **kwargs): + def __init__(self, start_date: int, account_id: str, **kwargs): super().__init__(**kwargs) self.account_id = account_id + self.start_date = start_date def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: decoded_response = response.json() @@ -58,9 +59,8 @@ class IncrementalStripeStream(StripeStream, ABC): # Stripe returns most recently created objects first, so we don't want to persist state until the entire stream has been read state_checkpoint_interval = math.inf - def __init__(self, start_date: str, lookback_window_days: int = 0, **kwargs): + def __init__(self, lookback_window_days: int = 0, **kwargs): super().__init__(**kwargs) - self.start_date = pendulum.parse(start_date).int_timestamp self.lookback_window_days = lookback_window_days @property @@ -146,7 +146,7 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs): return f"customers/{customer_id}/balance_transactions" def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: - customers_stream = Customers(authenticator=self.authenticator, account_id=self.account_id) + customers_stream = Customers(authenticator=self.authenticator, account_id=self.account_id, start_date=self.start_date) for customer in customers_stream.read_records(sync_mode=SyncMode.full_refresh): yield from super().read_records(stream_slice={"customer_id": customer["id"]}, **kwargs) @@ -206,7 +206,7 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs): return f"invoices/{stream_slice['invoice_id']}/lines" def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: - invoices_stream = Invoices(authenticator=self.authenticator, account_id=self.account_id) + invoices_stream = Invoices(authenticator=self.authenticator, account_id=self.account_id, start_date=self.start_date) for invoice in invoices_stream.read_records(sync_mode=SyncMode.full_refresh): yield from super().read_records(stream_slice={"invoice_id": invoice["id"]}, **kwargs) @@ -290,7 +290,7 @@ def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs): return params def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: - subscriptions_stream = Subscriptions(authenticator=self.authenticator, account_id=self.account_id) + subscriptions_stream = Subscriptions(authenticator=self.authenticator, account_id=self.account_id, start_date=self.start_date) for subscriptions in subscriptions_stream.read_records(sync_mode=SyncMode.full_refresh): yield from super().read_records(stream_slice={"subscription_id": subscriptions["id"]}, **kwargs) @@ -345,6 +345,6 @@ def request_params(self, **kwargs) -> MutableMapping[str, Any]: return params def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: - customers_stream = Customers(authenticator=self.authenticator, account_id=self.account_id) + customers_stream = Customers(authenticator=self.authenticator, account_id=self.account_id, start_date=self.start_date) for customer in customers_stream.read_records(sync_mode=SyncMode.full_refresh): yield from super().read_records(stream_slice={"customer_id": customer["id"]}, **kwargs) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py index f3560254ac6a..e517662202f6 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py @@ -22,6 +22,6 @@ ], ) def test_lookback_window(lookback_window_days, current_state, expected, message): - inv_stream = Invoices(account_id=213, start_date="2020", lookback_window_days=lookback_window_days) + inv_stream = Invoices(account_id=213, start_date=1577836800, lookback_window_days=lookback_window_days) inv_stream.cursor_field = "created" assert inv_stream.get_start_timestamp({"created": current_state}) == expected, message diff --git a/docs/integrations/sources/stripe.md b/docs/integrations/sources/stripe.md index 01f790eb624b..3e498ee27632 100644 --- a/docs/integrations/sources/stripe.md +++ b/docs/integrations/sources/stripe.md @@ -71,6 +71,7 @@ If you would like to test Airbyte using test data on Stripe, `sk_test_` and `rk_ | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | +| 0.1.21 | 2021-10-07 | [6841](https://github.com/airbytehq/airbyte/pull/6841) | Fix missing `start_date` argument + update json files for SAT | | 0.1.20 | 2021-09-30 | [6017](https://github.com/airbytehq/airbyte/pull/6017) | Add lookback_window_days parameter | | 0.1.19 | 2021-09-27 | [6466](https://github.com/airbytehq/airbyte/pull/6466) | Use `start_date` parameter in incremental streams | | 0.1.18 | 2021-09-14 | [6004](https://github.com/airbytehq/airbyte/pull/6004) | Fix coupons and subscriptions stream schemas by removing incorrect timestamp formatting |