🎉 Stripe Source: Fix subscriptions stream to return all kinds of subscriptions (including expired and canceled)

.github/workflows/publish-command.yml

@@ -126,7 +126,7 @@ jobs:
           SOURCE_SQUARE_CREDS: ${{ secrets.SOURCE_SQUARE_CREDS }}
           SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG: ${{ secrets.SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG }}
           SOURCE_RECURLY_INTEGRATION_TEST_CREDS: ${{ secrets.SOURCE_RECURLY_INTEGRATION_TEST_CREDS }}
-          STRIPE_INTEGRATION_TEST_CREDS: ${{ secrets.STRIPE_INTEGRATION_TEST_CREDS }}
+          SOURCE_STRIPE_CREDS: ${{ secrets.SOURCE_STRIPE_CREDS }}
           STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS: ${{ secrets.STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS }}
           SURVEYMONKEY_TEST_CREDS: ${{ secrets.SURVEYMONKEY_TEST_CREDS }}
           TEMPO_INTEGRATION_TEST_CREDS: ${{ secrets.TEMPO_INTEGRATION_TEST_CREDS }}

.github/workflows/test-command.yml

@@ -124,7 +124,7 @@ jobs:
           SOURCE_SQUARE_CREDS: ${{ secrets.SOURCE_SQUARE_CREDS }}
           SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG: ${{ secrets.SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG }}
           SOURCE_RECURLY_INTEGRATION_TEST_CREDS: ${{ secrets.SOURCE_RECURLY_INTEGRATION_TEST_CREDS }}
-          STRIPE_INTEGRATION_TEST_CREDS: ${{ secrets.STRIPE_INTEGRATION_TEST_CREDS }}
+          SOURCE_STRIPE_CREDS: ${{ secrets.SOURCE_STRIPE_CREDS }}
           STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS: ${{ secrets.STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS }}
           SURVEYMONKEY_TEST_CREDS: ${{ secrets.SURVEYMONKEY_TEST_CREDS }}
           TEMPO_INTEGRATION_TEST_CREDS: ${{ secrets.TEMPO_INTEGRATION_TEST_CREDS }}

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e094cb9a-26de-4645-8761-65c0c425d1de.json

@@ -2,7 +2,7 @@
   "sourceDefinitionId": "e094cb9a-26de-4645-8761-65c0c425d1de",
   "name": "Stripe",
   "dockerRepository": "airbyte/source-stripe",
-  "dockerImageTag": "0.1.13",
+  "dockerImageTag": "0.1.14",
   "documentationUrl": "https://hub.docker.com/r/airbyte/source-stripe",
   "icon": "stripe.svg"
 }

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -98,7 +98,7 @@
 - sourceDefinitionId: e094cb9a-26de-4645-8761-65c0c425d1de
   name: Stripe
   dockerRepository: airbyte/source-stripe
-  dockerImageTag: 0.1.13
+  dockerImageTag: 0.1.14
   documentationUrl: https://hub.docker.com/r/airbyte/source-stripe
   icon: stripe.svg
 - sourceDefinitionId: b03a9f3e-22a5-11eb-adc1-0242ac120002

airbyte-integrations/connectors/source-stripe/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.13
+LABEL io.airbyte.version=0.1.14
 LABEL io.airbyte.name=airbyte/source-stripe

airbyte-integrations/connectors/source-stripe/README.md

@@ -49,10 +49,10 @@ and place them into `secrets/config.json`.
 
 ### Locally running the connector
 ```
-python main_dev.py spec
-python main_dev.py check --config secrets/config.json
-python main_dev.py discover --config secrets/config.json
-python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json
+python main.py spec
+python main.py check --config secrets/config.json
+python main.py discover --config secrets/config.json
+python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json
 ```
 
 ### Unit Tests
@@ -61,17 +61,37 @@ To run unit tests locally, from the connector directory run:
 python -m pytest unit_tests
 ```
 
-### Locally running the connector docker image
+#### Acceptance Tests
+Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](source-acceptance-tests.md) for more information.
+If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
+To run your integration tests with acceptance tests, from the connector root, run
+```
+python -m pytest integration_tests -p integration_tests.acceptance
+```
+
+### Using gradle to run tests
+All commands should be run from airbyte project root.
+To run unit tests:
+```
+./gradlew :airbyte-integrations:connectors:source-stripe:unitTest
+```
+
+To run acceptance and custom integration tests:
+```
+./gradlew :airbyte-integrations:connectors:source-stripe:integrationTest
+```
 
 #### Build
+To run your integration tests with docker localy
+
 First, make sure you build the latest Docker image:
 ```
-docker build . -t airbyte/source-stripe:dev
+docker build --no-cache . -t airbyte/source-stripe:dev
 ```
 
 You can also build the connector image via Gradle:
 ```
-./gradlew :airbyte-integrations:connectors:source-stripe:airbyteDocker
+./gradlew clean :airbyte-integrations:connectors:source-stripe:airbyteDocker
 ```
 When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in
 the Dockerfile.

airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml

@@ -13,26 +13,34 @@ tests:
     - config_path: "secrets/config.json"
     - config_path: "secrets/connected_account_config.json"
   basic_read:
+    # TEST 1 - Reading catalog without invoice_line_items
+    # Along with this test we expect subscriptions with status in ["active","canceled"]
+    # If this test fails for some reason, please check the expected_subscriptions_records.json for valid subset of data.
     - config_path: "secrets/config.json"
-#     Reading a invoice_line_items stream takes too long and falls on timeout.
       configured_catalog_path: "integration_tests/non_invoice_line_items_catalog.json"
       validate_output_from_all_streams: yes
+      timeout_seconds: 3600
+      expect_records:
+        path: "integration_tests/expected_subscriptions_records.txt"
+    # TEST 2 - Reading data from account that has no records for stream Disputes
     - config_path: "secrets/connected_account_config.json"
-#     This account has no records for stream Disputes.
-      configured_catalog_path: "integration_tests/non_disputes_catalog.json"
+      configured_catalog_path: "integration_tests/non_disputes_events_catalog.json"
       validate_output_from_all_streams: yes
+      timeout_seconds: 3600
   incremental:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/non_invoice_line_items_catalog.json"
       future_state_path: "integration_tests/abnormal_state.json"
       cursor_paths:
-        charges: [ "created" ]
+        charges: ["created"]
     - config_path: "secrets/connected_account_config.json"
-      configured_catalog_path: "integration_tests/non_disputes_catalog.json"
+      configured_catalog_path: "integration_tests/non_disputes_events_catalog.json"
       future_state_path: "integration_tests/abnormal_state.json"
       cursor_paths:
-        charges: [ "created" ]
+        charges: ["created"]
   full_refresh:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/non_invoice_line_items_catalog.json"
+      timeout_seconds: 3600
     - config_path: "secrets/connected_account_config.json"
+

airbyte-integrations/connectors/source-stripe/build.gradle

@@ -1,29 +1,9 @@
 plugins {
     id 'airbyte-python'
     id 'airbyte-docker'
-    id 'airbyte-standard-source-test-file'
+    id 'airbyte-source-acceptance-test'
 }
 
 airbytePython {
     moduleDirectory 'source_stripe'
 }
-
-airbyteStandardSourceTestFile {
-    // For more information on standard source tests, see https://docs.airbyte.io/contributing-to-airbyte/building-new-connector/testing-connectors
-    // All these input paths must live inside this connector's directory (or subdirectories)
-
-    specPath = "source_stripe/spec.json"
-
-    // configPath points to a config file which matches the spec.json supplied above. secrets/ is gitignored by default, so place your config file
-    // there (in case it contains any credentials)
-    configPath = "secrets/config.json"
-
-    // Note: If your source supports incremental syncing, then make sure that the catalog that is returned in the get_catalog method is configured
-    // for incremental syncing (e.g. include cursor fields, etc).
-    configuredCatalogPath = "sample_files/configured_catalog.json"
-}
-
-
-dependencies {
-    implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs)
-}

airbyte-integrations/connectors/source-stripe/integration_tests/configured_catalog.json

@@ -0,0 +1,53 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "events",
+        "json_schema": {
+          "type": "object",
+          "properties": {
+            "created": {
+              "type": ["null", "integer"]
+            },
+            "data": {
+              "type": ["null", "object"],
+              "properties": {}
+            },
+            "id": {
+              "type": ["null", "string"]
+            },
+            "api_version": {
+              "type": ["null", "string"]
+            },
+            "object": {
+              "type": ["null", "string"]
+            },
+            "livemode": {
+              "type": ["null", "boolean"]
+            },
+            "pending_webhooks": {
+              "type": ["null", "integer"]
+            },
+            "request": {
+              "type": ["null", "string"]
+            },
+            "type": {
+              "type": ["null", "string"]
+            },
+            "updated": {
+              "type": ["null", "string"],
+              "format": "date-time"
+            }
+          }
+        },
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["created"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite",
+      "cursor_field": ["created"]
+    }
+  ]
+}