Skip to content

Commit

Permalink
🎉 Stripe Source: Fix subscriptions stream to return all kinds of subs…
Browse files Browse the repository at this point in the history
…criptions (including expired and canceled) (#4669)

#4669 Stripe Source: Fix subscriptions stream to return all kinds of subscriptions (including expired and canceled)
Co-authored-by: Oleksandr Bazarnov <oleksandr.bazarnov@globallogic.com>
  • Loading branch information
annalvova05 authored Jul 16, 2021
1 parent 7f4315f commit 28239a8
Show file tree
Hide file tree
Showing 34 changed files with 3,491 additions and 399 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/publish-command.yml
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ jobs:
SOURCE_SQUARE_CREDS: ${{ secrets.SOURCE_SQUARE_CREDS }}
SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG: ${{ secrets.SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG }}
SOURCE_RECURLY_INTEGRATION_TEST_CREDS: ${{ secrets.SOURCE_RECURLY_INTEGRATION_TEST_CREDS }}
STRIPE_INTEGRATION_TEST_CREDS: ${{ secrets.STRIPE_INTEGRATION_TEST_CREDS }}
SOURCE_STRIPE_CREDS: ${{ secrets.SOURCE_STRIPE_CREDS }}
STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS: ${{ secrets.STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS }}
SURVEYMONKEY_TEST_CREDS: ${{ secrets.SURVEYMONKEY_TEST_CREDS }}
TEMPO_INTEGRATION_TEST_CREDS: ${{ secrets.TEMPO_INTEGRATION_TEST_CREDS }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/test-command.yml
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ jobs:
SOURCE_SQUARE_CREDS: ${{ secrets.SOURCE_SQUARE_CREDS }}
SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG: ${{ secrets.SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG }}
SOURCE_RECURLY_INTEGRATION_TEST_CREDS: ${{ secrets.SOURCE_RECURLY_INTEGRATION_TEST_CREDS }}
STRIPE_INTEGRATION_TEST_CREDS: ${{ secrets.STRIPE_INTEGRATION_TEST_CREDS }}
SOURCE_STRIPE_CREDS: ${{ secrets.SOURCE_STRIPE_CREDS }}
STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS: ${{ secrets.STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS }}
SURVEYMONKEY_TEST_CREDS: ${{ secrets.SURVEYMONKEY_TEST_CREDS }}
TEMPO_INTEGRATION_TEST_CREDS: ${{ secrets.TEMPO_INTEGRATION_TEST_CREDS }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"sourceDefinitionId": "e094cb9a-26de-4645-8761-65c0c425d1de",
"name": "Stripe",
"dockerRepository": "airbyte/source-stripe",
"dockerImageTag": "0.1.13",
"dockerImageTag": "0.1.14",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-stripe",
"icon": "stripe.svg"
}
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@
- sourceDefinitionId: e094cb9a-26de-4645-8761-65c0c425d1de
name: Stripe
dockerRepository: airbyte/source-stripe
dockerImageTag: 0.1.13
dockerImageTag: 0.1.14
documentationUrl: https://hub.docker.com/r/airbyte/source-stripe
icon: stripe.svg
- sourceDefinitionId: b03a9f3e-22a5-11eb-adc1-0242ac120002
Expand Down
4 changes: 0 additions & 4 deletions airbyte-integrations/connectors/source-stripe/CHANGELOG.md

This file was deleted.

2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-stripe/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,5 @@ RUN pip install .
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]

LABEL io.airbyte.version=0.1.13
LABEL io.airbyte.version=0.1.14
LABEL io.airbyte.name=airbyte/source-stripe
34 changes: 27 additions & 7 deletions airbyte-integrations/connectors/source-stripe/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,10 @@ and place them into `secrets/config.json`.

### Locally running the connector
```
python main_dev.py spec
python main_dev.py check --config secrets/config.json
python main_dev.py discover --config secrets/config.json
python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json
python main.py spec
python main.py check --config secrets/config.json
python main.py discover --config secrets/config.json
python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json
```

### Unit Tests
Expand All @@ -61,17 +61,37 @@ To run unit tests locally, from the connector directory run:
python -m pytest unit_tests
```

### Locally running the connector docker image
#### Acceptance Tests
Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](source-acceptance-tests.md) for more information.
If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
To run your integration tests with acceptance tests, from the connector root, run
```
python -m pytest integration_tests -p integration_tests.acceptance
```

### Using gradle to run tests
All commands should be run from airbyte project root.
To run unit tests:
```
./gradlew :airbyte-integrations:connectors:source-stripe:unitTest
```

To run acceptance and custom integration tests:
```
./gradlew :airbyte-integrations:connectors:source-stripe:integrationTest
```

#### Build
To run your integration tests with docker localy

First, make sure you build the latest Docker image:
```
docker build . -t airbyte/source-stripe:dev
docker build --no-cache . -t airbyte/source-stripe:dev
```

You can also build the connector image via Gradle:
```
./gradlew :airbyte-integrations:connectors:source-stripe:airbyteDocker
./gradlew clean :airbyte-integrations:connectors:source-stripe:airbyteDocker
```
When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in
the Dockerfile.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,26 +13,34 @@ tests:
- config_path: "secrets/config.json"
- config_path: "secrets/connected_account_config.json"
basic_read:
# TEST 1 - Reading catalog without invoice_line_items
# Along with this test we expect subscriptions with status in ["active","canceled"]
# If this test fails for some reason, please check the expected_subscriptions_records.json for valid subset of data.
- config_path: "secrets/config.json"
# Reading a invoice_line_items stream takes too long and falls on timeout.
configured_catalog_path: "integration_tests/non_invoice_line_items_catalog.json"
validate_output_from_all_streams: yes
timeout_seconds: 3600
expect_records:
path: "integration_tests/expected_subscriptions_records.txt"
# TEST 2 - Reading data from account that has no records for stream Disputes
- config_path: "secrets/connected_account_config.json"
# This account has no records for stream Disputes.
configured_catalog_path: "integration_tests/non_disputes_catalog.json"
configured_catalog_path: "integration_tests/non_disputes_events_catalog.json"
validate_output_from_all_streams: yes
timeout_seconds: 3600
incremental:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/non_invoice_line_items_catalog.json"
future_state_path: "integration_tests/abnormal_state.json"
cursor_paths:
charges: [ "created" ]
charges: ["created"]
- config_path: "secrets/connected_account_config.json"
configured_catalog_path: "integration_tests/non_disputes_catalog.json"
configured_catalog_path: "integration_tests/non_disputes_events_catalog.json"
future_state_path: "integration_tests/abnormal_state.json"
cursor_paths:
charges: [ "created" ]
charges: ["created"]
full_refresh:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/non_invoice_line_items_catalog.json"
timeout_seconds: 3600
- config_path: "secrets/connected_account_config.json"

22 changes: 1 addition & 21 deletions airbyte-integrations/connectors/source-stripe/build.gradle
Original file line number Diff line number Diff line change
@@ -1,29 +1,9 @@
plugins {
id 'airbyte-python'
id 'airbyte-docker'
id 'airbyte-standard-source-test-file'
id 'airbyte-source-acceptance-test'
}

airbytePython {
moduleDirectory 'source_stripe'
}

airbyteStandardSourceTestFile {
// For more information on standard source tests, see https://docs.airbyte.io/contributing-to-airbyte/building-new-connector/testing-connectors
// All these input paths must live inside this connector's directory (or subdirectories)

specPath = "source_stripe/spec.json"

// configPath points to a config file which matches the spec.json supplied above. secrets/ is gitignored by default, so place your config file
// there (in case it contains any credentials)
configPath = "secrets/config.json"

// Note: If your source supports incremental syncing, then make sure that the catalog that is returned in the get_catalog method is configured
// for incremental syncing (e.g. include cursor fields, etc).
configuredCatalogPath = "sample_files/configured_catalog.json"
}


dependencies {
implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
{
"streams": [
{
"stream": {
"name": "events",
"json_schema": {
"type": "object",
"properties": {
"created": {
"type": ["null", "integer"]
},
"data": {
"type": ["null", "object"],
"properties": {}
},
"id": {
"type": ["null", "string"]
},
"api_version": {
"type": ["null", "string"]
},
"object": {
"type": ["null", "string"]
},
"livemode": {
"type": ["null", "boolean"]
},
"pending_webhooks": {
"type": ["null", "integer"]
},
"request": {
"type": ["null", "string"]
},
"type": {
"type": ["null", "string"]
},
"updated": {
"type": ["null", "string"],
"format": "date-time"
}
}
},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["created"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "overwrite",
"cursor_field": ["created"]
}
]
}
Loading

0 comments on commit 28239a8

Please sign in to comment.