From 052d5fdc09ceb850dcccbbcd9f41beed00a1fed0 Mon Sep 17 00:00:00 2001 From: Foluso Ogunlana Date: Sat, 9 Oct 2021 14:50:45 +0100 Subject: [PATCH 01/22] feat(67): add support for 'spec' using Python HTTP API source template and stripe as an example --- .../connectors/source-paystack/.dockerignore | 7 + .../connectors/source-paystack/Dockerfile | 38 ++++ .../connectors/source-paystack/README.md | 132 +++++++++++ .../acceptance-test-config.yml | 30 +++ .../source-paystack/acceptance-test-docker.sh | 16 ++ .../connectors/source-paystack/build.gradle | 14 ++ .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 5 + .../integration_tests/acceptance.py | 16 ++ .../integration_tests/catalog.json | 39 ++++ .../integration_tests/configured_catalog.json | 22 ++ .../integration_tests/invalid_config.json | 3 + .../integration_tests/sample_config.json | 3 + .../integration_tests/sample_state.json | 5 + .../connectors/source-paystack/main.py | 13 ++ .../source-paystack/requirements.txt | 2 + .../connectors/source-paystack/setup.py | 29 +++ .../source_paystack/__init__.py | 8 + .../source_paystack/schemas/TODO.md | 25 +++ .../source_paystack/schemas/customers.json | 16 ++ .../source_paystack/schemas/employees.json | 19 ++ .../source-paystack/source_paystack/source.py | 206 ++++++++++++++++++ .../source-paystack/source_paystack/spec.json | 31 +++ .../source-paystack/unit_tests/__init__.py | 3 + .../unit_tests/test_incremental_streams.py | 59 +++++ .../source-paystack/unit_tests/test_source.py | 22 ++ .../unit_tests/test_streams.py | 83 +++++++ docs/integrations/sources/paystack.md | 87 ++++++++ 28 files changed, 936 insertions(+) create mode 100644 airbyte-integrations/connectors/source-paystack/.dockerignore create mode 100644 airbyte-integrations/connectors/source-paystack/Dockerfile create mode 100644 airbyte-integrations/connectors/source-paystack/README.md create mode 100644 airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-paystack/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-paystack/build.gradle create mode 100644 airbyte-integrations/connectors/source-paystack/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-paystack/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-paystack/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-paystack/integration_tests/catalog.json create mode 100644 airbyte-integrations/connectors/source-paystack/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-paystack/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-paystack/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-paystack/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-paystack/main.py create mode 100644 airbyte-integrations/connectors/source-paystack/requirements.txt create mode 100644 airbyte-integrations/connectors/source-paystack/setup.py create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/__init__.py create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/schemas/TODO.md create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/schemas/employees.json create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/source.py create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/spec.json create mode 100644 airbyte-integrations/connectors/source-paystack/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-paystack/unit_tests/test_incremental_streams.py create mode 100644 airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py create mode 100644 docs/integrations/sources/paystack.md diff --git a/airbyte-integrations/connectors/source-paystack/.dockerignore b/airbyte-integrations/connectors/source-paystack/.dockerignore new file mode 100644 index 000000000000..65d4e9d2cf76 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/.dockerignore @@ -0,0 +1,7 @@ +* +!Dockerfile +!Dockerfile.test +!main.py +!source_paystack +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-paystack/Dockerfile b/airbyte-integrations/connectors/source-paystack/Dockerfile new file mode 100644 index 000000000000..dc7a3b8f443d --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.7.11-alpine3.14 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_paystack ./source_paystack + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-paystack diff --git a/airbyte-integrations/connectors/source-paystack/README.md b/airbyte-integrations/connectors/source-paystack/README.md new file mode 100644 index 000000000000..377059e585c0 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/README.md @@ -0,0 +1,132 @@ +# Paystack Source + +This is the repository for the Paystack source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/paystack). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-paystack:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/paystack) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_paystack/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source paystack test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-paystack:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-paystack:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-paystack:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-paystack:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-paystack:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-paystack:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-paystack:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-paystack:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml b/airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml new file mode 100644 index 000000000000..a198287e8f01 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-paystack:dev +tests: + spec: + - spec_path: "source_paystack/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.txt" +# extra_fields: no +# exact_order: no +# extra_records: yes + incremental: # TODO if your connector does not implement incremental sync, remove this block + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-paystack/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-paystack/acceptance-test-docker.sh new file mode 100644 index 000000000000..e4d8b1cef896 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-paystack/build.gradle b/airbyte-integrations/connectors/source-paystack/build.gradle new file mode 100644 index 000000000000..dd725aa404c4 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/build.gradle @@ -0,0 +1,14 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_paystack' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) + implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/__init__.py b/airbyte-integrations/connectors/source-paystack/integration_tests/__init__.py new file mode 100644 index 000000000000..46b7376756ec --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-paystack/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-paystack/integration_tests/acceptance.py new file mode 100644 index 000000000000..58c194c5d137 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """ This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/catalog.json b/airbyte-integrations/connectors/source-paystack/integration_tests/catalog.json new file mode 100644 index 000000000000..6799946a6851 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/catalog.json @@ -0,0 +1,39 @@ +{ + "streams": [ + { + "name": "TODO fix this file", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": "column1", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + }, + { + "name": "table1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-paystack/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..36f0468db0d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/configured_catalog.json @@ -0,0 +1,22 @@ +{ + "streams": [ + { + "stream": { + "name": "customers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "employees", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-paystack/integration_tests/invalid_config.json new file mode 100644 index 000000000000..f3732995784f --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "todo-wrong-field": "this should be an incomplete config file, used in standard tests" +} diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-paystack/integration_tests/sample_config.json new file mode 100644 index 000000000000..ecc4913b84c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "fix-me": "TODO" +} diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-paystack/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-paystack/main.py b/airbyte-integrations/connectors/source-paystack/main.py new file mode 100644 index 000000000000..a524a86f134a --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_paystack import SourcePaystack + +if __name__ == "__main__": + source = SourcePaystack() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-paystack/requirements.txt b/airbyte-integrations/connectors/source-paystack/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-paystack/setup.py b/airbyte-integrations/connectors/source-paystack/setup.py new file mode 100644 index 000000000000..1c11aae3ef04 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_paystack", + description="Source implementation for Paystack.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/__init__.py b/airbyte-integrations/connectors/source-paystack/source_paystack/__init__.py new file mode 100644 index 000000000000..84b9299483ef --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from .source import SourcePaystack + +__all__ = ["SourcePaystack"] diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/TODO.md b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/TODO.md new file mode 100644 index 000000000000..cf1efadb3c9c --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/TODO.md @@ -0,0 +1,25 @@ +# TODO: Define your stream schemas +Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). + +The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. + +The schema of a stream is the return value of `Stream.get_json_schema`. + +## Static schemas +By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. + +Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. + +## Dynamic schemas +If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). + +## Dynamically modifying static schemas +Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: +``` +def get_json_schema(self): + schema = super().get_json_schema() + schema['dynamically_determined_property'] = "property" + return schema +``` + +Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json new file mode 100644 index 000000000000..9a4b13485836 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "signup_date": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/employees.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/employees.json new file mode 100644 index 000000000000..2fa01a0fa1ff --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/employees.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "years_of_service": { + "type": ["null", "integer"] + }, + "start_date": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/source.py b/airbyte-integrations/connectors/source-paystack/source_paystack/source.py new file mode 100644 index 000000000000..809e39ed4d39 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/source.py @@ -0,0 +1,206 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + +""" +TODO: Most comments in this class are instructive and should be deleted after the source is implemented. + +This file provides a stubbed example of how to use the Airbyte CDK to develop both a source connector which supports full refresh or and an +incremental syncs from an HTTP API. + +The various TODOs are both implementation hints and steps - fulfilling all the TODOs should be sufficient to implement one basic and one incremental +stream from a source. This pattern is the same one used by Airbyte internally to implement connectors. + +The approach here is not authoritative, and devs are free to use their own judgement. + +There are additional required TODOs in the files within the integration_tests folder and the spec.json file. +""" + + +# Basic full refresh stream +class PaystackStream(HttpStream, ABC): + """ + TODO remove this comment + + This class represents a stream output by the connector. + This is an abstract base class meant to contain all the common functionality at the API level e.g: the API base URL, pagination strategy, + parsing responses etc.. + + Each stream should extend this class (or another abstract subclass of it) to specify behavior unique to that stream. + + Typically for REST APIs each stream corresponds to a resource in the API. For example if the API + contains the endpoints + - GET v1/customers + - GET v1/employees + + then you should have three classes: + `class PaystackStream(HttpStream, ABC)` which is the current class + `class Customers(PaystackStream)` contains behavior to pull data for customers using v1/customers + `class Employees(PaystackStream)` contains behavior to pull data for employees using v1/employees + + If some streams implement incremental sync, it is typical to create another class + `class IncrementalPaystackStream((PaystackStream), ABC)` then have concrete stream implementations extend it. An example + is provided below. + + See the reference docs for the full list of configurable options. + """ + + # TODO: Fill in the url base. Required. + url_base = "https://example-api.com/v1/" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + TODO: Override this method to define a pagination strategy. If you will not be using pagination, no action is required - just return None. + + This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed + to most other methods in this class to help you form headers, request bodies, query params, etc.. + + For example, if the API accepts a 'page' parameter to determine which page of the result to return, and a response from the API contains a + 'page' number, then this method should probably return a dict {'page': response.json()['page'] + 1} to increment the page count by 1. + The request_params method should then read the input next_page_token and set the 'page' param to next_page_token['page']. + + :param response: the most recent response from the API + :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. + If there are no more pages in the result, return None. + """ + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + """ + TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. + Usually contains common params e.g. pagination size etc. + """ + return {} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + TODO: Override this method to define how a response is parsed. + :return an iterable containing each record in the response + """ + yield {} + + +class Customers(PaystackStream): + """ + TODO: Change class name to match the table/data source this stream corresponds to. + """ + + # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. + primary_key = "customer_id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + """ + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this + should return "customers". Required. + """ + return "customers" + + +# Basic incremental stream +class IncrementalPaystackStream(PaystackStream, ABC): + """ + TODO fill in details of this class to implement functionality related to incremental syncs for your connector. + if you do not need to implement incremental sync for any streams, remove this class. + """ + + # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. + state_checkpoint_interval = None + + @property + def cursor_field(self) -> str: + """ + TODO + Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is + usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. + + :return str: The name of the cursor field. + """ + return [] + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and + the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. + """ + return {} + + +class Employees(IncrementalPaystackStream): + """ + TODO: Change class name to match the table/data source this stream corresponds to. + """ + + # TODO: Fill in the cursor_field. Required. + cursor_field = "start_date" + + # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. + primary_key = "employee_id" + + def path(self, **kwargs) -> str: + """ + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/employees then this should + return "single". Required. + """ + return "employees" + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + """ + TODO: Optionally override this method to define this stream's slices. If slicing is not needed, delete this method. + + Slices control when state is saved. Specifically, state is saved after a slice has been fully read. + This is useful if the API offers reads by groups or filters, and can be paired with the state object to make reads efficient. See the "concepts" + section of the docs for more information. + + The function is called before reading any records in a stream. It returns an Iterable of dicts, each containing the + necessary data to craft a request for a slice. The stream state is usually referenced to determine what slices need to be created. + This means that data in a slice is usually closely related to a stream's cursor_field and stream_state. + + An HTTP request is made for each returned slice. The same slice can be accessed in the path, request_params and request_header functions to help + craft that specific request. + + For example, if https://example-api.com/v1/employees offers a date query params that returns data for that particular day, one way to implement + this would be to consult the stream state object for the last synced date, then return a slice containing each date from the last synced date + till now. The request_params function would then grab the date from the stream_slice and make it part of the request by injecting it into + the date query param. + """ + raise NotImplementedError("Implement stream slices or delete this method!") + + +# Source +class SourcePaystack(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + """ + TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API + + See https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py#L232 + for an example. + + :param config: the user-input config object conforming to the connector's spec.json + :param logger: logger object + :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. + """ + return True, None + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + TODO: Replace the streams below with your own streams. + + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + # TODO remove the authenticator if not required. + auth = TokenAuthenticator(token="api_key") # Oauth2Authenticator is also available if you need oauth support + return [Customers(authenticator=auth), Employees(authenticator=auth)] diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/spec.json b/airbyte-integrations/connectors/source-paystack/source_paystack/spec.json new file mode 100644 index 000000000000..9ee4c587ecbb --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/spec.json @@ -0,0 +1,31 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/paystack", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Paystack Source Spec", + "type": "object", + "required": ["secret_key", "start_date"], + "additionalProperties": false, + "properties": { + "secret_key": { + "type": "string", + "pattern": "^(s|r)k_(live|test)_[a-zA-Z0-9]+$", + "description": "Paystack API key (usually starts with 'sk_live_'; find yours here).", + "airbyte_secret": true + }, + "start_date": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", + "examples": ["2017-01-25T00:00:00Z"] + }, + "lookback_window_days": { + "type": "integer", + "title": "Lookback Window (in days)", + "default": 0, + "minimum": 0, + "description": "When set, the connector will always reload data from the past N days, where N is the value set here. This is useful if your data is updated after creation." + } + } + } +} diff --git a/airbyte-integrations/connectors/source-paystack/unit_tests/__init__.py b/airbyte-integrations/connectors/source-paystack/unit_tests/__init__.py new file mode 100644 index 000000000000..46b7376756ec --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-paystack/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-paystack/unit_tests/test_incremental_streams.py new file mode 100644 index 000000000000..4fea097e0358 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/unit_tests/test_incremental_streams.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from airbyte_cdk.models import SyncMode +from pytest import fixture +from source_paystack.source import IncrementalPaystackStream + + +@fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(IncrementalPaystackStream, "path", "v0/example_endpoint") + mocker.patch.object(IncrementalPaystackStream, "primary_key", "test_primary_key") + mocker.patch.object(IncrementalPaystackStream, "__abstractmethods__", set()) + + +def test_cursor_field(patch_incremental_base_class): + stream = IncrementalPaystackStream() + # TODO: replace this with your expected cursor field + expected_cursor_field = [] + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(patch_incremental_base_class): + stream = IncrementalPaystackStream() + # TODO: replace this with your input parameters + inputs = {"current_stream_state": None, "latest_record": None} + # TODO: replace this with your expected updated stream state + expected_state = {} + assert stream.get_updated_state(**inputs) == expected_state + + +def test_stream_slices(patch_incremental_base_class): + stream = IncrementalPaystackStream() + # TODO: replace this with your input parameters + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} + # TODO: replace this with your expected stream slices list + expected_stream_slice = [None] + assert stream.stream_slices(**inputs) == expected_stream_slice + + +def test_supports_incremental(patch_incremental_base_class, mocker): + mocker.patch.object(IncrementalPaystackStream, "cursor_field", "dummy_field") + stream = IncrementalPaystackStream() + assert stream.supports_incremental + + +def test_source_defined_cursor(patch_incremental_base_class): + stream = IncrementalPaystackStream() + assert stream.source_defined_cursor + + +def test_stream_checkpoint_interval(patch_incremental_base_class): + stream = IncrementalPaystackStream() + # TODO: replace this with your expected checkpoint interval + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py b/airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py new file mode 100644 index 000000000000..a8ecdf1f3a11 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_paystack.source import SourcePaystack + + +def test_check_connection(mocker): + source = SourcePaystack() + logger_mock, config_mock = MagicMock(), MagicMock() + assert source.check_connection(logger_mock, config_mock) == (True, None) + + +def test_streams(mocker): + source = SourcePaystack() + config_mock = MagicMock() + streams = source.streams(config_mock) + # TODO: replace this with your streams number + expected_streams_number = 2 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py new file mode 100644 index 000000000000..7ecdde9868e1 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_paystack.source import PaystackStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(PaystackStream, "path", "v0/example_endpoint") + mocker.patch.object(PaystackStream, "primary_key", "test_primary_key") + mocker.patch.object(PaystackStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = PaystackStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request parameters + expected_params = {} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = PaystackStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected next page token + expected_token = None + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class): + stream = PaystackStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected parced object + expected_parsed_object = {} + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = PaystackStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request headers + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = PaystackStream() + # TODO: replace this with your expected http request method + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = PaystackStream() + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = PaystackStream() + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/docs/integrations/sources/paystack.md b/docs/integrations/sources/paystack.md new file mode 100644 index 000000000000..0bdd580a40d1 --- /dev/null +++ b/docs/integrations/sources/paystack.md @@ -0,0 +1,87 @@ +# Stripe + +## Overview + +The Paystack source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. + +### Output schema + +This Source is capable of syncing the following core Streams: + + + +### Note on Incremental Syncs + +The Paystack API does not allow querying objects which were updated since the last sync. Therefore, this connector uses the `created_at` field to query for new data in your Paystack account. + +If your data is updated after creation, you can use the Loockback Window option when configuring the connector to always reload data from the past N days. This will allow you to pick up updates to the data. + +### Data type mapping + + + +### Features + +| Feature | Supported? | +| :--- | :--- | + + +### Performance considerations + + + +## Getting started + +### Requirements + +* Paystack API Secret Key + +### Setup guide + +Visit the [Paystack dashboard settings page](https://dashboard.paystack.com/#/settings/developer) with developer level access or more to see the secret key for your account. Secret keys for the live Paystack environment will be prefixed with `sk_live_`. + +Unfortunately Paystack does not yet support restricted permission levels on secret keys. This means that you will have to use the same secret key here that you use for charging customers. Use at your own risk. In the future Paystack might support restricted access levels and in that case Airbyte only requires a read-only access level key. + +If you would like to test Airbyte using test data on Paystack, `sk_test_` API keys are also supported. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :--- | :--- | :--- | :--- | + From d256b81b2c5c1421e25f1a31c983934d0c59c0fe Mon Sep 17 00:00:00 2001 From: Foluso Ogunlana Date: Sat, 9 Oct 2021 15:01:00 +0100 Subject: [PATCH 02/22] chore(67): add sample state and config --- .../connectors/source-paystack/sample_files/config.json | 4 ++++ .../connectors/source-paystack/sample_files/state.json | 4 ++++ 2 files changed, 8 insertions(+) create mode 100644 airbyte-integrations/connectors/source-paystack/sample_files/config.json create mode 100644 airbyte-integrations/connectors/source-paystack/sample_files/state.json diff --git a/airbyte-integrations/connectors/source-paystack/sample_files/config.json b/airbyte-integrations/connectors/source-paystack/sample_files/config.json new file mode 100644 index 000000000000..39ea4ff8007c --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/sample_files/config.json @@ -0,0 +1,4 @@ +{ + "secret_key": "sk_test(live)_", + "start_date": "2020-05-01T00:00:00Z" +} diff --git a/airbyte-integrations/connectors/source-paystack/sample_files/state.json b/airbyte-integrations/connectors/source-paystack/sample_files/state.json new file mode 100644 index 000000000000..fb73d59f510e --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/sample_files/state.json @@ -0,0 +1,4 @@ +{ + "transactions": { "created": 1617030403 } +} + \ No newline at end of file From 6da85d36266d2f683de28773a2407cedb514e889 Mon Sep 17 00:00:00 2001 From: Foluso Ogunlana Date: Sat, 9 Oct 2021 17:48:16 +0100 Subject: [PATCH 03/22] feat(67): add check functionality for paystack source by fetching first customer --- .../source-paystack/sample_files/config.json | 2 +- .../source-paystack/source_paystack/source.py | 198 ++---------------- 2 files changed, 23 insertions(+), 177 deletions(-) diff --git a/airbyte-integrations/connectors/source-paystack/sample_files/config.json b/airbyte-integrations/connectors/source-paystack/sample_files/config.json index 39ea4ff8007c..d1d95ecae175 100644 --- a/airbyte-integrations/connectors/source-paystack/sample_files/config.json +++ b/airbyte-integrations/connectors/source-paystack/sample_files/config.json @@ -1,4 +1,4 @@ { - "secret_key": "sk_test(live)_", + "secret_key": "sk_test_abc123", "start_date": "2020-05-01T00:00:00Z" } diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/source.py b/airbyte-integrations/connectors/source-paystack/source_paystack/source.py index 809e39ed4d39..e34c7001cf8b 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/source.py +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/source.py @@ -1,8 +1,6 @@ # # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # - - from abc import ABC from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple @@ -12,187 +10,37 @@ from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -""" -TODO: Most comments in this class are instructive and should be deleted after the source is implemented. - -This file provides a stubbed example of how to use the Airbyte CDK to develop both a source connector which supports full refresh or and an -incremental syncs from an HTTP API. - -The various TODOs are both implementation hints and steps - fulfilling all the TODOs should be sufficient to implement one basic and one incremental -stream from a source. This pattern is the same one used by Airbyte internally to implement connectors. - -The approach here is not authoritative, and devs are free to use their own judgement. - -There are additional required TODOs in the files within the integration_tests folder and the spec.json file. -""" - - -# Basic full refresh stream -class PaystackStream(HttpStream, ABC): - """ - TODO remove this comment - - This class represents a stream output by the connector. - This is an abstract base class meant to contain all the common functionality at the API level e.g: the API base URL, pagination strategy, - parsing responses etc.. - - Each stream should extend this class (or another abstract subclass of it) to specify behavior unique to that stream. - - Typically for REST APIs each stream corresponds to a resource in the API. For example if the API - contains the endpoints - - GET v1/customers - - GET v1/employees - - then you should have three classes: - `class PaystackStream(HttpStream, ABC)` which is the current class - `class Customers(PaystackStream)` contains behavior to pull data for customers using v1/customers - `class Employees(PaystackStream)` contains behavior to pull data for employees using v1/employees - - If some streams implement incremental sync, it is typical to create another class - `class IncrementalPaystackStream((PaystackStream), ABC)` then have concrete stream implementations extend it. An example - is provided below. - - See the reference docs for the full list of configurable options. - """ - - # TODO: Fill in the url base. Required. - url_base = "https://example-api.com/v1/" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - TODO: Override this method to define a pagination strategy. If you will not be using pagination, no action is required - just return None. - - This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed - to most other methods in this class to help you form headers, request bodies, query params, etc.. - - For example, if the API accepts a 'page' parameter to determine which page of the result to return, and a response from the API contains a - 'page' number, then this method should probably return a dict {'page': response.json()['page'] + 1} to increment the page count by 1. - The request_params method should then read the input next_page_token and set the 'page' param to next_page_token['page']. - - :param response: the most recent response from the API - :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. - If there are no more pages in the result, return None. - """ - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - """ - TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. - Usually contains common params e.g. pagination size etc. - """ - return {} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - TODO: Override this method to define how a response is parsed. - :return an iterable containing each record in the response - """ - yield {} - - -class Customers(PaystackStream): - """ - TODO: Change class name to match the table/data source this stream corresponds to. - """ - - # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. - primary_key = "customer_id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this - should return "customers". Required. - """ - return "customers" - - -# Basic incremental stream -class IncrementalPaystackStream(PaystackStream, ABC): - """ - TODO fill in details of this class to implement functionality related to incremental syncs for your connector. - if you do not need to implement incremental sync for any streams, remove this class. - """ - - # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. - state_checkpoint_interval = None - - @property - def cursor_field(self) -> str: - """ - TODO - Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is - usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. - - :return str: The name of the cursor field. - """ - return [] - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and - the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. - """ - return {} - - -class Employees(IncrementalPaystackStream): - """ - TODO: Change class name to match the table/data source this stream corresponds to. - """ - - # TODO: Fill in the cursor_field. Required. - cursor_field = "start_date" - - # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. - primary_key = "employee_id" - - def path(self, **kwargs) -> str: - """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/employees then this should - return "single". Required. - """ - return "employees" - - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - """ - TODO: Optionally override this method to define this stream's slices. If slicing is not needed, delete this method. - - Slices control when state is saved. Specifically, state is saved after a slice has been fully read. - This is useful if the API offers reads by groups or filters, and can be paired with the state object to make reads efficient. See the "concepts" - section of the docs for more information. - - The function is called before reading any records in a stream. It returns an Iterable of dicts, each containing the - necessary data to craft a request for a slice. The stream state is usually referenced to determine what slices need to be created. - This means that data in a slice is usually closely related to a stream's cursor_field and stream_state. - - An HTTP request is made for each returned slice. The same slice can be accessed in the path, request_params and request_header functions to help - craft that specific request. - - For example, if https://example-api.com/v1/employees offers a date query params that returns data for that particular day, one way to implement - this would be to consult the stream state object for the last synced date, then return a slice containing each date from the last synced date - till now. The request_params function would then grab the date from the stream_slice and make it part of the request by injecting it into - the date query param. - """ - raise NotImplementedError("Implement stream slices or delete this method!") - +PAYSTACK_API_BASE_URL = "https://api.paystack.co/" # Source class SourcePaystack(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: """ - TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API - - See https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py#L232 - for an example. + Check connection by fetching customers :param config: the user-input config object conforming to the connector's spec.json :param logger: logger object :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ + + try: + response = requests.get( + f"{PAYSTACK_API_BASE_URL}/customer?page=1&perPage=1", + headers={"Authorization": f"Bearer {config['secret_key']}"}, + verify=True + ) + response.raise_for_status() + except Exception as e: + msg = e + if e.response.status_code == 401: + msg = 'Connection to Paystack could not be authorized. Please check your secret key.' + + return False, msg + + response_json = response.json() + if response_json['status'] == False: + return False, 'Connection test failed due to error: ' + response_json['message'] + return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: @@ -201,6 +49,4 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: :param config: A Mapping of the user input configuration as defined in the connector spec. """ - # TODO remove the authenticator if not required. - auth = TokenAuthenticator(token="api_key") # Oauth2Authenticator is also available if you need oauth support - return [Customers(authenticator=auth), Employees(authenticator=auth)] + return [] From e02aeccac8f5539d083b3bc4bb5c505638c0182a Mon Sep 17 00:00:00 2001 From: Foluso Ogunlana Date: Sun, 10 Oct 2021 19:24:23 +0100 Subject: [PATCH 04/22] feat(67): add support for discover and read customer stream --- .../source-paystack/sample_files/config.json | 2 +- .../sample_files/configured_catalog.json | 57 +++++++++ .../source_paystack/constants.py | 3 + .../source_paystack/schemas/customers.json | 40 +++++- .../source_paystack/schemas/employees.json | 19 --- .../source-paystack/source_paystack/source.py | 25 ++-- .../source_paystack/streams.py | 117 ++++++++++++++++++ 7 files changed, 225 insertions(+), 38 deletions(-) create mode 100644 airbyte-integrations/connectors/source-paystack/sample_files/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/constants.py delete mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/schemas/employees.json create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/streams.py diff --git a/airbyte-integrations/connectors/source-paystack/sample_files/config.json b/airbyte-integrations/connectors/source-paystack/sample_files/config.json index d1d95ecae175..6e33d086d3dc 100644 --- a/airbyte-integrations/connectors/source-paystack/sample_files/config.json +++ b/airbyte-integrations/connectors/source-paystack/sample_files/config.json @@ -1,4 +1,4 @@ { "secret_key": "sk_test_abc123", - "start_date": "2020-05-01T00:00:00Z" + "start_date": "2020-08-01T00:00:00Z" } diff --git a/airbyte-integrations/connectors/source-paystack/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-paystack/sample_files/configured_catalog.json new file mode 100644 index 000000000000..51e670ae21f7 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/sample_files/configured_catalog.json @@ -0,0 +1,57 @@ +{ + "streams": [ + { + "stream": { + "name": "customers", + "json_schema": { + "type": ["null", "object"], + "properties": { + "integration": { + "type": ["null", "integer"] + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "domain": { + "type": ["null", "string"] + }, + "customer_code": { + "type": ["null", "string"] + }, + "risk_action": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/constants.py b/airbyte-integrations/connectors/source-paystack/source_paystack/constants.py new file mode 100644 index 000000000000..f3412c1210a9 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/constants.py @@ -0,0 +1,3 @@ +PAYSTACK_API_BASE_URL = "https://api.paystack.co/" +AUTH_KEY_FIELD = "secret_key" +PAYSTACK_CREATED_AT = "createdAt" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json index 9a4b13485836..7cc9020ad0a4 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json @@ -1,16 +1,44 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", + "type": ["null", "object"], "properties": { - "id": { + "integration": { + "type": ["null", "integer"] + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "domain": { "type": ["null", "string"] }, - "name": { + "customer_code": { "type": ["null", "string"] }, - "signup_date": { + "risk_action": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedAt": { "type": ["null", "string"], "format": "date-time" } - } + } } diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/employees.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/employees.json deleted file mode 100644 index 2fa01a0fa1ff..000000000000 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/employees.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "years_of_service": { - "type": ["null", "integer"] - }, - "start_date": { - "type": ["null", "string"], - "format": "date-time" - } - } -} diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/source.py b/airbyte-integrations/connectors/source-paystack/source_paystack/source.py index e34c7001cf8b..707ca6499707 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/source.py +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/source.py @@ -1,18 +1,15 @@ # # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from typing import Any, List, Mapping, Tuple import requests from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from source_paystack.constants import PAYSTACK_API_BASE_URL, AUTH_KEY_FIELD +from source_paystack.streams import Customers -PAYSTACK_API_BASE_URL = "https://api.paystack.co/" - -# Source class SourcePaystack(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: """ @@ -26,27 +23,31 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: try: response = requests.get( f"{PAYSTACK_API_BASE_URL}/customer?page=1&perPage=1", - headers={"Authorization": f"Bearer {config['secret_key']}"}, + headers={"Authorization": f"Bearer {config[AUTH_KEY_FIELD]}"}, verify=True ) response.raise_for_status() except Exception as e: msg = e if e.response.status_code == 401: - msg = 'Connection to Paystack could not be authorized. Please check your secret key.' - + msg = 'Connection to Paystack was not authorized. Please check that your secret key is correct.' return False, msg response_json = response.json() if response_json['status'] == False: return False, 'Connection test failed due to error: ' + response_json['message'] - return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ - TODO: Replace the streams below with your own streams. + Returns list of streams output by the Paystack source connector :param config: A Mapping of the user input configuration as defined in the connector spec. """ - return [] + authenticator = TokenAuthenticator(config[AUTH_KEY_FIELD]) + args = {"authenticator": authenticator, "start_date": config["start_date"]} + incremental_args = {**args, "lookback_window_days": config.get("lookback_window_days")} + return [ + Customers(**incremental_args) + ] + diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py b/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py new file mode 100644 index 000000000000..4d5ab98fac70 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py @@ -0,0 +1,117 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# +import math +from abc import ABC, abstractmethod +from typing import Any, Iterable, Mapping, MutableMapping, Optional +from datetime import datetime + +import requests +from airbyte_cdk.sources.streams.http import HttpStream +from source_paystack.constants import PAYSTACK_API_BASE_URL, PAYSTACK_CREATED_AT + + +class PaystackStream(HttpStream, ABC): + url_base = PAYSTACK_API_BASE_URL + primary_key = "id" + + def __init__(self, start_date: int, **kwargs): + super().__init__(**kwargs) + self.start_date = start_date + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + decoded_response = response.json() + page, pageCount = decoded_response['meta']['page'], response.json()['meta']['pageCount'] + + if page != pageCount: + return { "page": page + 1 } + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, any] = None, + next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + + params = {"perPage": 200} + if next_page_token: + params.update(next_page_token) + + return params + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + response_json = response.json() + yield from response_json.get("data", []) # Paystack puts records in a container array "data" + + +class IncrementalPaystackStream(PaystackStream, ABC): + # Paystack (like Stripe) returns most recently created objects first, so we don't want to persist state until the entire stream has been read + state_checkpoint_interval = math.inf + + def __init__(self, lookback_window_days: int = 0, **kwargs): + super().__init__(**kwargs) + self.lookback_window_days = lookback_window_days + + @property + @abstractmethod + def cursor_field(self) -> str: + """ + Defining a cursor field indicates that a stream is incremental, so any incremental stream must extend this class + and define a cursor field. + """ + pass + + def get_updated_state( + self, + current_stream_state: MutableMapping[str, Any], + latest_record: Mapping[str, Any] + ) -> Mapping[str, Any]: + """ + Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object + and returning an updated state object. + """ + return { + self.cursor_field: max( + latest_record.get(self.cursor_field), + current_stream_state.get(self.cursor_field, None), + key=lambda d: pendulum.parse(d).int_timestamp if d else 0 + ) + } + + def request_params(self, stream_state: Mapping[str, Any] = None, **kwargs): + stream_state = stream_state or {} + params = super().request_params(stream_state=stream_state, **kwargs) + + start_timestamp = self.get_start_timestamp(stream_state) + if start_timestamp: + params["from"] = start_timestamp + return params + + def get_start_timestamp(self, stream_state) -> str: + start_point = self.start_date + if stream_state and self.cursor_field in stream_state: + start_point = max( + start_point, + stream_state[self.cursor_field], + key=lambda d: pendulum.parse(d).int_timestamp + ) + + if start_point and self.lookback_window_days: + self.logger.info(f"Applying lookback window of {self.lookback_window_days} days to stream {self.name}") + start_point = pendulum.parse(start_point)\ + .subtract(days=abs(self.lookback_window_days))\ + .isoformat()\ + .replace("+00:00", "Z") + + return start_point + + +class Customers(IncrementalPaystackStream): + """ + API docs: https://paystack.com/docs/api/#customer-list + """ + + cursor_field = PAYSTACK_CREATED_AT + + def path(self, **kwargs) -> str: + return "customer" \ No newline at end of file From 7a90777cd3264c29e80bee0da1343d39236ace1d Mon Sep 17 00:00:00 2001 From: Foluso Ogunlana Date: Sun, 10 Oct 2021 19:43:25 +0100 Subject: [PATCH 05/22] feat(67): add paystack source connector to UI --- airbyte-config/init/src/main/resources/icons/paystack.svg | 1 + .../init/src/main/resources/seed/source_definitions.yaml | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 airbyte-config/init/src/main/resources/icons/paystack.svg diff --git a/airbyte-config/init/src/main/resources/icons/paystack.svg b/airbyte-config/init/src/main/resources/icons/paystack.svg new file mode 100644 index 000000000000..6cf03a4a71d1 --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/paystack.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index d6ea388add06..55fc9c5459b4 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -130,6 +130,13 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/stripe icon: stripe.svg sourceType: api +- sourceDefinitionId: 193bdcb8-1dd9-48d1-aade-91cadfd74f9b + name: Paystack + dockerRepository: airbyte/source-paystack + dockerImageTag: dev + documentationUrl: https://docs.airbyte.io/integrations/sources/paystack + icon: paystack.svg + sourceType: api - sourceDefinitionId: b03a9f3e-22a5-11eb-adc1-0242ac120002 name: Mailchimp dockerRepository: airbyte/source-mailchimp From 0babd5fe912b5f87de1cd3c2ada62f299523df20 Mon Sep 17 00:00:00 2001 From: Foluso Ogunlana Date: Mon, 11 Oct 2021 10:18:58 +0100 Subject: [PATCH 06/22] feat(67): update source definitions to use 0.1.0 --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 74911fbc97f4..4a16950e4b49 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -133,7 +133,7 @@ - sourceDefinitionId: 193bdcb8-1dd9-48d1-aade-91cadfd74f9b name: Paystack dockerRepository: airbyte/source-paystack - dockerImageTag: dev + dockerImageTag: 0.1.0 documentationUrl: https://docs.airbyte.io/integrations/sources/paystack icon: paystack.svg sourceType: api From c37347c4121f4e1af71737edba6416f163322d74 Mon Sep 17 00:00:00 2001 From: Foluso <5675998+foogunlana@users.noreply.github.com> Date: Mon, 11 Oct 2021 10:26:09 +0100 Subject: [PATCH 07/22] Hacktoberfest 67 paystack source (#1) * feat(67): add support for 'spec' using Python HTTP API source template and stripe as an example * chore(67): add sample state and config * feat(67): add check functionality for paystack source by fetching first customer * feat(67): add support for discover and read customer stream * feat(67): add paystack source connector to UI * feat(67): update source definitions to use 0.1.0 Co-authored-by: Foluso Ogunlana --- .../src/main/resources/icons/paystack.svg | 1 + .../resources/seed/source_definitions.yaml | 7 + .../connectors/source-paystack/.dockerignore | 7 + .../connectors/source-paystack/Dockerfile | 38 +++++ .../connectors/source-paystack/README.md | 132 ++++++++++++++++++ .../acceptance-test-config.yml | 30 ++++ .../source-paystack/acceptance-test-docker.sh | 16 +++ .../connectors/source-paystack/build.gradle | 14 ++ .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 5 + .../integration_tests/acceptance.py | 16 +++ .../integration_tests/catalog.json | 39 ++++++ .../integration_tests/configured_catalog.json | 22 +++ .../integration_tests/invalid_config.json | 3 + .../integration_tests/sample_config.json | 3 + .../integration_tests/sample_state.json | 5 + .../connectors/source-paystack/main.py | 13 ++ .../source-paystack/requirements.txt | 2 + .../source-paystack/sample_files/config.json | 4 + .../sample_files/configured_catalog.json | 57 ++++++++ .../source-paystack/sample_files/state.json | 4 + .../connectors/source-paystack/setup.py | 29 ++++ .../source_paystack/__init__.py | 8 ++ .../source_paystack/constants.py | 3 + .../source_paystack/schemas/TODO.md | 25 ++++ .../source_paystack/schemas/customers.json | 44 ++++++ .../source-paystack/source_paystack/source.py | 53 +++++++ .../source-paystack/source_paystack/spec.json | 31 ++++ .../source_paystack/streams.py | 117 ++++++++++++++++ .../source-paystack/unit_tests/__init__.py | 3 + .../unit_tests/test_incremental_streams.py | 59 ++++++++ .../source-paystack/unit_tests/test_source.py | 22 +++ .../unit_tests/test_streams.py | 83 +++++++++++ docs/integrations/sources/paystack.md | 87 ++++++++++++ 34 files changed, 985 insertions(+) create mode 100644 airbyte-config/init/src/main/resources/icons/paystack.svg create mode 100644 airbyte-integrations/connectors/source-paystack/.dockerignore create mode 100644 airbyte-integrations/connectors/source-paystack/Dockerfile create mode 100644 airbyte-integrations/connectors/source-paystack/README.md create mode 100644 airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-paystack/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-paystack/build.gradle create mode 100644 airbyte-integrations/connectors/source-paystack/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-paystack/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-paystack/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-paystack/integration_tests/catalog.json create mode 100644 airbyte-integrations/connectors/source-paystack/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-paystack/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-paystack/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-paystack/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-paystack/main.py create mode 100644 airbyte-integrations/connectors/source-paystack/requirements.txt create mode 100644 airbyte-integrations/connectors/source-paystack/sample_files/config.json create mode 100644 airbyte-integrations/connectors/source-paystack/sample_files/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-paystack/sample_files/state.json create mode 100644 airbyte-integrations/connectors/source-paystack/setup.py create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/__init__.py create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/constants.py create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/schemas/TODO.md create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/source.py create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/spec.json create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/streams.py create mode 100644 airbyte-integrations/connectors/source-paystack/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-paystack/unit_tests/test_incremental_streams.py create mode 100644 airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py create mode 100644 docs/integrations/sources/paystack.md diff --git a/airbyte-config/init/src/main/resources/icons/paystack.svg b/airbyte-config/init/src/main/resources/icons/paystack.svg new file mode 100644 index 000000000000..6cf03a4a71d1 --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/paystack.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 8dc8aa1bbc60..4a16950e4b49 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -130,6 +130,13 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/stripe icon: stripe.svg sourceType: api +- sourceDefinitionId: 193bdcb8-1dd9-48d1-aade-91cadfd74f9b + name: Paystack + dockerRepository: airbyte/source-paystack + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/paystack + icon: paystack.svg + sourceType: api - sourceDefinitionId: b03a9f3e-22a5-11eb-adc1-0242ac120002 name: Mailchimp dockerRepository: airbyte/source-mailchimp diff --git a/airbyte-integrations/connectors/source-paystack/.dockerignore b/airbyte-integrations/connectors/source-paystack/.dockerignore new file mode 100644 index 000000000000..65d4e9d2cf76 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/.dockerignore @@ -0,0 +1,7 @@ +* +!Dockerfile +!Dockerfile.test +!main.py +!source_paystack +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-paystack/Dockerfile b/airbyte-integrations/connectors/source-paystack/Dockerfile new file mode 100644 index 000000000000..dc7a3b8f443d --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.7.11-alpine3.14 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_paystack ./source_paystack + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-paystack diff --git a/airbyte-integrations/connectors/source-paystack/README.md b/airbyte-integrations/connectors/source-paystack/README.md new file mode 100644 index 000000000000..377059e585c0 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/README.md @@ -0,0 +1,132 @@ +# Paystack Source + +This is the repository for the Paystack source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/paystack). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-paystack:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/paystack) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_paystack/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source paystack test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-paystack:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-paystack:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-paystack:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-paystack:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-paystack:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-paystack:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-paystack:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-paystack:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml b/airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml new file mode 100644 index 000000000000..a198287e8f01 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-paystack:dev +tests: + spec: + - spec_path: "source_paystack/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.txt" +# extra_fields: no +# exact_order: no +# extra_records: yes + incremental: # TODO if your connector does not implement incremental sync, remove this block + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-paystack/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-paystack/acceptance-test-docker.sh new file mode 100644 index 000000000000..e4d8b1cef896 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-paystack/build.gradle b/airbyte-integrations/connectors/source-paystack/build.gradle new file mode 100644 index 000000000000..dd725aa404c4 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/build.gradle @@ -0,0 +1,14 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_paystack' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) + implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/__init__.py b/airbyte-integrations/connectors/source-paystack/integration_tests/__init__.py new file mode 100644 index 000000000000..46b7376756ec --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-paystack/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-paystack/integration_tests/acceptance.py new file mode 100644 index 000000000000..58c194c5d137 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """ This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/catalog.json b/airbyte-integrations/connectors/source-paystack/integration_tests/catalog.json new file mode 100644 index 000000000000..6799946a6851 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/catalog.json @@ -0,0 +1,39 @@ +{ + "streams": [ + { + "name": "TODO fix this file", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": "column1", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + }, + { + "name": "table1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-paystack/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..36f0468db0d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/configured_catalog.json @@ -0,0 +1,22 @@ +{ + "streams": [ + { + "stream": { + "name": "customers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "employees", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-paystack/integration_tests/invalid_config.json new file mode 100644 index 000000000000..f3732995784f --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "todo-wrong-field": "this should be an incomplete config file, used in standard tests" +} diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-paystack/integration_tests/sample_config.json new file mode 100644 index 000000000000..ecc4913b84c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "fix-me": "TODO" +} diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-paystack/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-paystack/main.py b/airbyte-integrations/connectors/source-paystack/main.py new file mode 100644 index 000000000000..a524a86f134a --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_paystack import SourcePaystack + +if __name__ == "__main__": + source = SourcePaystack() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-paystack/requirements.txt b/airbyte-integrations/connectors/source-paystack/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-paystack/sample_files/config.json b/airbyte-integrations/connectors/source-paystack/sample_files/config.json new file mode 100644 index 000000000000..6e33d086d3dc --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/sample_files/config.json @@ -0,0 +1,4 @@ +{ + "secret_key": "sk_test_abc123", + "start_date": "2020-08-01T00:00:00Z" +} diff --git a/airbyte-integrations/connectors/source-paystack/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-paystack/sample_files/configured_catalog.json new file mode 100644 index 000000000000..51e670ae21f7 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/sample_files/configured_catalog.json @@ -0,0 +1,57 @@ +{ + "streams": [ + { + "stream": { + "name": "customers", + "json_schema": { + "type": ["null", "object"], + "properties": { + "integration": { + "type": ["null", "integer"] + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "domain": { + "type": ["null", "string"] + }, + "customer_code": { + "type": ["null", "string"] + }, + "risk_action": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paystack/sample_files/state.json b/airbyte-integrations/connectors/source-paystack/sample_files/state.json new file mode 100644 index 000000000000..fb73d59f510e --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/sample_files/state.json @@ -0,0 +1,4 @@ +{ + "transactions": { "created": 1617030403 } +} + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paystack/setup.py b/airbyte-integrations/connectors/source-paystack/setup.py new file mode 100644 index 000000000000..1c11aae3ef04 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_paystack", + description="Source implementation for Paystack.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/__init__.py b/airbyte-integrations/connectors/source-paystack/source_paystack/__init__.py new file mode 100644 index 000000000000..84b9299483ef --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from .source import SourcePaystack + +__all__ = ["SourcePaystack"] diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/constants.py b/airbyte-integrations/connectors/source-paystack/source_paystack/constants.py new file mode 100644 index 000000000000..f3412c1210a9 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/constants.py @@ -0,0 +1,3 @@ +PAYSTACK_API_BASE_URL = "https://api.paystack.co/" +AUTH_KEY_FIELD = "secret_key" +PAYSTACK_CREATED_AT = "createdAt" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/TODO.md b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/TODO.md new file mode 100644 index 000000000000..cf1efadb3c9c --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/TODO.md @@ -0,0 +1,25 @@ +# TODO: Define your stream schemas +Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). + +The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. + +The schema of a stream is the return value of `Stream.get_json_schema`. + +## Static schemas +By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. + +Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. + +## Dynamic schemas +If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). + +## Dynamically modifying static schemas +Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: +``` +def get_json_schema(self): + schema = super().get_json_schema() + schema['dynamically_determined_property'] = "property" + return schema +``` + +Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json new file mode 100644 index 000000000000..7cc9020ad0a4 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json @@ -0,0 +1,44 @@ +{ + "type": ["null", "object"], + "properties": { + "integration": { + "type": ["null", "integer"] + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "domain": { + "type": ["null", "string"] + }, + "customer_code": { + "type": ["null", "string"] + }, + "risk_action": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/source.py b/airbyte-integrations/connectors/source-paystack/source_paystack/source.py new file mode 100644 index 000000000000..707ca6499707 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/source.py @@ -0,0 +1,53 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# +from typing import Any, List, Mapping, Tuple + +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from source_paystack.constants import PAYSTACK_API_BASE_URL, AUTH_KEY_FIELD +from source_paystack.streams import Customers + +class SourcePaystack(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + """ + Check connection by fetching customers + + :param config: the user-input config object conforming to the connector's spec.json + :param logger: logger object + :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. + """ + + try: + response = requests.get( + f"{PAYSTACK_API_BASE_URL}/customer?page=1&perPage=1", + headers={"Authorization": f"Bearer {config[AUTH_KEY_FIELD]}"}, + verify=True + ) + response.raise_for_status() + except Exception as e: + msg = e + if e.response.status_code == 401: + msg = 'Connection to Paystack was not authorized. Please check that your secret key is correct.' + return False, msg + + response_json = response.json() + if response_json['status'] == False: + return False, 'Connection test failed due to error: ' + response_json['message'] + return True, None + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + Returns list of streams output by the Paystack source connector + + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + authenticator = TokenAuthenticator(config[AUTH_KEY_FIELD]) + args = {"authenticator": authenticator, "start_date": config["start_date"]} + incremental_args = {**args, "lookback_window_days": config.get("lookback_window_days")} + return [ + Customers(**incremental_args) + ] + diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/spec.json b/airbyte-integrations/connectors/source-paystack/source_paystack/spec.json new file mode 100644 index 000000000000..9ee4c587ecbb --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/spec.json @@ -0,0 +1,31 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/paystack", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Paystack Source Spec", + "type": "object", + "required": ["secret_key", "start_date"], + "additionalProperties": false, + "properties": { + "secret_key": { + "type": "string", + "pattern": "^(s|r)k_(live|test)_[a-zA-Z0-9]+$", + "description": "Paystack API key (usually starts with 'sk_live_'; find yours here).", + "airbyte_secret": true + }, + "start_date": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", + "examples": ["2017-01-25T00:00:00Z"] + }, + "lookback_window_days": { + "type": "integer", + "title": "Lookback Window (in days)", + "default": 0, + "minimum": 0, + "description": "When set, the connector will always reload data from the past N days, where N is the value set here. This is useful if your data is updated after creation." + } + } + } +} diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py b/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py new file mode 100644 index 000000000000..4d5ab98fac70 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py @@ -0,0 +1,117 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# +import math +from abc import ABC, abstractmethod +from typing import Any, Iterable, Mapping, MutableMapping, Optional +from datetime import datetime + +import requests +from airbyte_cdk.sources.streams.http import HttpStream +from source_paystack.constants import PAYSTACK_API_BASE_URL, PAYSTACK_CREATED_AT + + +class PaystackStream(HttpStream, ABC): + url_base = PAYSTACK_API_BASE_URL + primary_key = "id" + + def __init__(self, start_date: int, **kwargs): + super().__init__(**kwargs) + self.start_date = start_date + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + decoded_response = response.json() + page, pageCount = decoded_response['meta']['page'], response.json()['meta']['pageCount'] + + if page != pageCount: + return { "page": page + 1 } + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, any] = None, + next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + + params = {"perPage": 200} + if next_page_token: + params.update(next_page_token) + + return params + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + response_json = response.json() + yield from response_json.get("data", []) # Paystack puts records in a container array "data" + + +class IncrementalPaystackStream(PaystackStream, ABC): + # Paystack (like Stripe) returns most recently created objects first, so we don't want to persist state until the entire stream has been read + state_checkpoint_interval = math.inf + + def __init__(self, lookback_window_days: int = 0, **kwargs): + super().__init__(**kwargs) + self.lookback_window_days = lookback_window_days + + @property + @abstractmethod + def cursor_field(self) -> str: + """ + Defining a cursor field indicates that a stream is incremental, so any incremental stream must extend this class + and define a cursor field. + """ + pass + + def get_updated_state( + self, + current_stream_state: MutableMapping[str, Any], + latest_record: Mapping[str, Any] + ) -> Mapping[str, Any]: + """ + Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object + and returning an updated state object. + """ + return { + self.cursor_field: max( + latest_record.get(self.cursor_field), + current_stream_state.get(self.cursor_field, None), + key=lambda d: pendulum.parse(d).int_timestamp if d else 0 + ) + } + + def request_params(self, stream_state: Mapping[str, Any] = None, **kwargs): + stream_state = stream_state or {} + params = super().request_params(stream_state=stream_state, **kwargs) + + start_timestamp = self.get_start_timestamp(stream_state) + if start_timestamp: + params["from"] = start_timestamp + return params + + def get_start_timestamp(self, stream_state) -> str: + start_point = self.start_date + if stream_state and self.cursor_field in stream_state: + start_point = max( + start_point, + stream_state[self.cursor_field], + key=lambda d: pendulum.parse(d).int_timestamp + ) + + if start_point and self.lookback_window_days: + self.logger.info(f"Applying lookback window of {self.lookback_window_days} days to stream {self.name}") + start_point = pendulum.parse(start_point)\ + .subtract(days=abs(self.lookback_window_days))\ + .isoformat()\ + .replace("+00:00", "Z") + + return start_point + + +class Customers(IncrementalPaystackStream): + """ + API docs: https://paystack.com/docs/api/#customer-list + """ + + cursor_field = PAYSTACK_CREATED_AT + + def path(self, **kwargs) -> str: + return "customer" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paystack/unit_tests/__init__.py b/airbyte-integrations/connectors/source-paystack/unit_tests/__init__.py new file mode 100644 index 000000000000..46b7376756ec --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-paystack/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-paystack/unit_tests/test_incremental_streams.py new file mode 100644 index 000000000000..4fea097e0358 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/unit_tests/test_incremental_streams.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from airbyte_cdk.models import SyncMode +from pytest import fixture +from source_paystack.source import IncrementalPaystackStream + + +@fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(IncrementalPaystackStream, "path", "v0/example_endpoint") + mocker.patch.object(IncrementalPaystackStream, "primary_key", "test_primary_key") + mocker.patch.object(IncrementalPaystackStream, "__abstractmethods__", set()) + + +def test_cursor_field(patch_incremental_base_class): + stream = IncrementalPaystackStream() + # TODO: replace this with your expected cursor field + expected_cursor_field = [] + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(patch_incremental_base_class): + stream = IncrementalPaystackStream() + # TODO: replace this with your input parameters + inputs = {"current_stream_state": None, "latest_record": None} + # TODO: replace this with your expected updated stream state + expected_state = {} + assert stream.get_updated_state(**inputs) == expected_state + + +def test_stream_slices(patch_incremental_base_class): + stream = IncrementalPaystackStream() + # TODO: replace this with your input parameters + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} + # TODO: replace this with your expected stream slices list + expected_stream_slice = [None] + assert stream.stream_slices(**inputs) == expected_stream_slice + + +def test_supports_incremental(patch_incremental_base_class, mocker): + mocker.patch.object(IncrementalPaystackStream, "cursor_field", "dummy_field") + stream = IncrementalPaystackStream() + assert stream.supports_incremental + + +def test_source_defined_cursor(patch_incremental_base_class): + stream = IncrementalPaystackStream() + assert stream.source_defined_cursor + + +def test_stream_checkpoint_interval(patch_incremental_base_class): + stream = IncrementalPaystackStream() + # TODO: replace this with your expected checkpoint interval + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py b/airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py new file mode 100644 index 000000000000..a8ecdf1f3a11 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_paystack.source import SourcePaystack + + +def test_check_connection(mocker): + source = SourcePaystack() + logger_mock, config_mock = MagicMock(), MagicMock() + assert source.check_connection(logger_mock, config_mock) == (True, None) + + +def test_streams(mocker): + source = SourcePaystack() + config_mock = MagicMock() + streams = source.streams(config_mock) + # TODO: replace this with your streams number + expected_streams_number = 2 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py new file mode 100644 index 000000000000..7ecdde9868e1 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_paystack.source import PaystackStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(PaystackStream, "path", "v0/example_endpoint") + mocker.patch.object(PaystackStream, "primary_key", "test_primary_key") + mocker.patch.object(PaystackStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = PaystackStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request parameters + expected_params = {} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = PaystackStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected next page token + expected_token = None + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class): + stream = PaystackStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected parced object + expected_parsed_object = {} + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = PaystackStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request headers + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = PaystackStream() + # TODO: replace this with your expected http request method + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = PaystackStream() + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = PaystackStream() + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/docs/integrations/sources/paystack.md b/docs/integrations/sources/paystack.md new file mode 100644 index 000000000000..0bdd580a40d1 --- /dev/null +++ b/docs/integrations/sources/paystack.md @@ -0,0 +1,87 @@ +# Stripe + +## Overview + +The Paystack source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. + +### Output schema + +This Source is capable of syncing the following core Streams: + + + +### Note on Incremental Syncs + +The Paystack API does not allow querying objects which were updated since the last sync. Therefore, this connector uses the `created_at` field to query for new data in your Paystack account. + +If your data is updated after creation, you can use the Loockback Window option when configuring the connector to always reload data from the past N days. This will allow you to pick up updates to the data. + +### Data type mapping + + + +### Features + +| Feature | Supported? | +| :--- | :--- | + + +### Performance considerations + + + +## Getting started + +### Requirements + +* Paystack API Secret Key + +### Setup guide + +Visit the [Paystack dashboard settings page](https://dashboard.paystack.com/#/settings/developer) with developer level access or more to see the secret key for your account. Secret keys for the live Paystack environment will be prefixed with `sk_live_`. + +Unfortunately Paystack does not yet support restricted permission levels on secret keys. This means that you will have to use the same secret key here that you use for charging customers. Use at your own risk. In the future Paystack might support restricted access levels and in that case Airbyte only requires a read-only access level key. + +If you would like to test Airbyte using test data on Paystack, `sk_test_` API keys are also supported. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :--- | :--- | :--- | :--- | + From fe0e5d96e17c5425a49f3dbd325b402f2878582a Mon Sep 17 00:00:00 2001 From: Foluso Ogunlana Date: Thu, 14 Oct 2021 08:25:13 +0100 Subject: [PATCH 08/22] feat(67): update stream state cursor field to be integer and to match API record field name --- .../source-paystack/sample_files/state.json | 3 +- .../source-paystack/source_paystack/source.py | 4 ++- .../source_paystack/streams.py | 29 ++++++++----------- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/airbyte-integrations/connectors/source-paystack/sample_files/state.json b/airbyte-integrations/connectors/source-paystack/sample_files/state.json index fb73d59f510e..2747c731e448 100644 --- a/airbyte-integrations/connectors/source-paystack/sample_files/state.json +++ b/airbyte-integrations/connectors/source-paystack/sample_files/state.json @@ -1,4 +1,3 @@ { - "transactions": { "created": 1617030403 } + "customers": { "createdAt": 1596240000 } } - \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/source.py b/airbyte-integrations/connectors/source-paystack/source_paystack/source.py index 707ca6499707..03d2db220f3a 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/source.py +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/source.py @@ -4,6 +4,7 @@ from typing import Any, List, Mapping, Tuple import requests +import pendulum from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator @@ -45,7 +46,8 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: :param config: A Mapping of the user input configuration as defined in the connector spec. """ authenticator = TokenAuthenticator(config[AUTH_KEY_FIELD]) - args = {"authenticator": authenticator, "start_date": config["start_date"]} + start_date = pendulum.parse(config["start_date"]).int_timestamp + args = {"authenticator": authenticator, "start_date": start_date} incremental_args = {**args, "lookback_window_days": config.get("lookback_window_days")} return [ Customers(**incremental_args) diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py b/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py index 4d5ab98fac70..05135f57ea52 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py @@ -7,6 +7,7 @@ from datetime import datetime import requests +import pendulum from airbyte_cdk.sources.streams.http import HttpStream from source_paystack.constants import PAYSTACK_API_BASE_URL, PAYSTACK_CREATED_AT @@ -70,11 +71,11 @@ def get_updated_state( Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object and returning an updated state object. """ + latest_record_created = latest_record.get(self.cursor_field) return { self.cursor_field: max( - latest_record.get(self.cursor_field), - current_stream_state.get(self.cursor_field, None), - key=lambda d: pendulum.parse(d).int_timestamp if d else 0 + pendulum.parse(latest_record_created).int_timestamp, + current_stream_state.get(self.cursor_field, 0), ) } @@ -82,28 +83,22 @@ def request_params(self, stream_state: Mapping[str, Any] = None, **kwargs): stream_state = stream_state or {} params = super().request_params(stream_state=stream_state, **kwargs) - start_timestamp = self.get_start_timestamp(stream_state) - if start_timestamp: - params["from"] = start_timestamp + start_date = self._get_start_date(stream_state) + if start_date: + params["from"] = start_date return params - def get_start_timestamp(self, stream_state) -> str: + def _get_start_date(self, stream_state) -> str: start_point = self.start_date if stream_state and self.cursor_field in stream_state: - start_point = max( - start_point, - stream_state[self.cursor_field], - key=lambda d: pendulum.parse(d).int_timestamp - ) + stream_record_created = stream_state[self.cursor_field] + start_point = max(start_point, stream_record_created) if start_point and self.lookback_window_days: self.logger.info(f"Applying lookback window of {self.lookback_window_days} days to stream {self.name}") - start_point = pendulum.parse(start_point)\ - .subtract(days=abs(self.lookback_window_days))\ - .isoformat()\ - .replace("+00:00", "Z") + start_point = pendulum.from_timestamp(start_point).subtract(days=abs(self.lookback_window_days)).int_timestamp - return start_point + return pendulum.from_timestamp(start_point).isoformat().replace("+00:00", "Z") class Customers(IncrementalPaystackStream): From 1225e2fa38990151036c16468949f9d3bcc10d19 Mon Sep 17 00:00:00 2001 From: Foluso Ogunlana Date: Tue, 19 Oct 2021 13:36:23 +0100 Subject: [PATCH 09/22] chore(67): add unit tests for source and streams --- .../connectors/source-paystack/setup.py | 1 + .../source-paystack/source_paystack/source.py | 8 +- .../source_paystack/streams.py | 8 +- .../unit_tests/test_incremental_streams.py | 135 +++++++++++++----- .../source-paystack/unit_tests/test_source.py | 28 +++- .../unit_tests/test_streams.py | 77 +++++----- 6 files changed, 168 insertions(+), 89 deletions(-) diff --git a/airbyte-integrations/connectors/source-paystack/setup.py b/airbyte-integrations/connectors/source-paystack/setup.py index 1c11aae3ef04..9f32194ae49f 100644 --- a/airbyte-integrations/connectors/source-paystack/setup.py +++ b/airbyte-integrations/connectors/source-paystack/setup.py @@ -13,6 +13,7 @@ "pytest~=6.1", "pytest-mock~=3.6.1", "source-acceptance-test", + "requests-mock" ] setup( diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/source.py b/airbyte-integrations/connectors/source-paystack/source_paystack/source.py index 03d2db220f3a..9018d07bd80b 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/source.py +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/source.py @@ -20,17 +20,19 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: :param logger: logger object :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ - try: response = requests.get( - f"{PAYSTACK_API_BASE_URL}/customer?page=1&perPage=1", + f"{PAYSTACK_API_BASE_URL}customer?page=1&perPage=1", headers={"Authorization": f"Bearer {config[AUTH_KEY_FIELD]}"}, verify=True ) response.raise_for_status() except Exception as e: msg = e - if e.response.status_code == 401: + if ( + isinstance(e, requests.exceptions.HTTPError) + and e.response.status_code == 401 + ): msg = 'Connection to Paystack was not authorized. Please check that your secret key is correct.' return False, msg diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py b/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py index 05135f57ea52..045fb1345cf5 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py @@ -41,8 +41,7 @@ def request_params( return params def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - response_json = response.json() - yield from response_json.get("data", []) # Paystack puts records in a container array "data" + yield from response.json().get("data", []) # Paystack puts records in a container array "data" class IncrementalPaystackStream(PaystackStream, ABC): @@ -82,10 +81,7 @@ def get_updated_state( def request_params(self, stream_state: Mapping[str, Any] = None, **kwargs): stream_state = stream_state or {} params = super().request_params(stream_state=stream_state, **kwargs) - - start_date = self._get_start_date(stream_state) - if start_date: - params["from"] = start_date + params["from"] = self._get_start_date(stream_state) return params def _get_start_date(self, stream_state) -> str: diff --git a/airbyte-integrations/connectors/source-paystack/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-paystack/unit_tests/test_incremental_streams.py index 4fea097e0358..657b9dfcbf2c 100644 --- a/airbyte-integrations/connectors/source-paystack/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-paystack/unit_tests/test_incremental_streams.py @@ -2,11 +2,17 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # +import uuid +import math +import pytest +import pendulum from airbyte_cdk.models import SyncMode from pytest import fixture -from source_paystack.source import IncrementalPaystackStream +from source_paystack.streams import IncrementalPaystackStream, Customers +# 1596240000 is equivalent to pendulum.parse("2020-08-01T00:00:00Z").int_timestamp +START_DATE = 1596240000 @fixture def patch_incremental_base_class(mocker): @@ -14,46 +20,99 @@ def patch_incremental_base_class(mocker): mocker.patch.object(IncrementalPaystackStream, "path", "v0/example_endpoint") mocker.patch.object(IncrementalPaystackStream, "primary_key", "test_primary_key") mocker.patch.object(IncrementalPaystackStream, "__abstractmethods__", set()) - - -def test_cursor_field(patch_incremental_base_class): - stream = IncrementalPaystackStream() - # TODO: replace this with your expected cursor field - expected_cursor_field = [] - assert stream.cursor_field == expected_cursor_field - - -def test_get_updated_state(patch_incremental_base_class): - stream = IncrementalPaystackStream() - # TODO: replace this with your input parameters - inputs = {"current_stream_state": None, "latest_record": None} - # TODO: replace this with your expected updated stream state - expected_state = {} - assert stream.get_updated_state(**inputs) == expected_state - - -def test_stream_slices(patch_incremental_base_class): - stream = IncrementalPaystackStream() - # TODO: replace this with your input parameters - inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} - # TODO: replace this with your expected stream slices list - expected_stream_slice = [None] - assert stream.stream_slices(**inputs) == expected_stream_slice - - -def test_supports_incremental(patch_incremental_base_class, mocker): - mocker.patch.object(IncrementalPaystackStream, "cursor_field", "dummy_field") - stream = IncrementalPaystackStream() + mocker.patch.object(IncrementalPaystackStream, "cursor_field", str(uuid.uuid4())) + + +def test_get_updated_state_uses_timestamp_of_latest_record(patch_incremental_base_class): + stream = IncrementalPaystackStream(start_date=START_DATE) + inputs = { + "current_stream_state": {stream.cursor_field: pendulum.parse("2021-08-01").int_timestamp}, + "latest_record": {stream.cursor_field: "2021-08-02T00:00:00Z"} + } + + updated_state = stream.get_updated_state(**inputs) + + assert updated_state == {stream.cursor_field: pendulum.parse("2021-08-02T00:00:00Z").int_timestamp} + +def test_get_updated_state_returns_current_state_for_old_records(patch_incremental_base_class): + stream = IncrementalPaystackStream(start_date=START_DATE) + current_state = {stream.cursor_field: pendulum.parse("2021-08-03").int_timestamp} + inputs = { + "current_stream_state": current_state, + "latest_record": {stream.cursor_field: "2021-08-02T00:00:00Z"} + } + + updated_state = stream.get_updated_state(**inputs) + + assert updated_state == current_state + +def test_get_updated_state_uses_timestamp_of_latest_record_when_no_current_state_exists(patch_incremental_base_class): + stream = IncrementalPaystackStream(start_date=START_DATE) + inputs = { + "current_stream_state": {}, + "latest_record": {stream.cursor_field: "2021-08-02T00:00:00Z"} + } + + updated_state = stream.get_updated_state(**inputs) + + assert updated_state == {stream.cursor_field: pendulum.parse("2021-08-02T00:00:00Z").int_timestamp} + +def test_request_params_includes_incremental_start_point(patch_incremental_base_class): + stream = IncrementalPaystackStream(start_date=START_DATE) + inputs = { + "stream_slice": None, + "next_page_token": {"page": 37}, + "stream_state": {stream.cursor_field: pendulum.parse("2021-09-02").int_timestamp}, + } + + params = stream.request_params(**inputs) + + assert params == {"perPage": 200, "page": 37, "from": "2021-09-02T00:00:00Z"} + +@pytest.mark.parametrize( + "lookback_window_days, current_state, expected, message", + [ + (None, "2021-08-30", "2021-08-30T00:00:00Z", "if lookback_window_days is not set should not affect cursor value"), + (0, "2021-08-30", "2021-08-30T00:00:00Z", "if lookback_window_days is not set should not affect cursor value"), + (10, "2021-08-30", "2021-08-20T00:00:00Z", "Should calculate cursor value as expected"), + (-10, "2021-08-30", "2021-08-20T00:00:00Z", "Should not care for the sign, use the module"), + ], +) +def test_request_params_incremental_start_point_applies_lookback_window( + patch_incremental_base_class, + lookback_window_days, + current_state, + expected, + message +): + stream = IncrementalPaystackStream(start_date=START_DATE, lookback_window_days=lookback_window_days) + inputs = { + "stream_slice": None, + "next_page_token": None, + "stream_state": {stream.cursor_field: pendulum.parse(current_state).int_timestamp}, + } + + params = stream.request_params(**inputs) + + assert params["perPage"] == 200 + assert params["from"] == expected, message + +def test_request_params_incremental_start_point_defaults_to_start_date(patch_incremental_base_class): + stream = IncrementalPaystackStream(start_date=START_DATE) + inputs = {"stream_slice": None, "next_page_token": None, "stream_state": None} + + params = stream.request_params(**inputs) + + assert params == {"perPage": 200, "from": "2020-08-01T00:00:00Z"} + +def test_supports_incremental(patch_incremental_base_class): + stream = IncrementalPaystackStream(start_date=START_DATE) assert stream.supports_incremental - def test_source_defined_cursor(patch_incremental_base_class): - stream = IncrementalPaystackStream() + stream = IncrementalPaystackStream(start_date=START_DATE) assert stream.source_defined_cursor - def test_stream_checkpoint_interval(patch_incremental_base_class): - stream = IncrementalPaystackStream() - # TODO: replace this with your expected checkpoint interval - expected_checkpoint_interval = None - assert stream.state_checkpoint_interval == expected_checkpoint_interval + stream = IncrementalPaystackStream(start_date=START_DATE) + assert stream.state_checkpoint_interval == math.inf \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py b/airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py index a8ecdf1f3a11..9964fa0708fc 100644 --- a/airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py @@ -2,21 +2,35 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # -from unittest.mock import MagicMock +from unittest.mock import MagicMock, ANY +import requests from source_paystack.source import SourcePaystack -def test_check_connection(mocker): +def test_check_connection_success(mocker, requests_mock): source = SourcePaystack() logger_mock, config_mock = MagicMock(), MagicMock() + requests_mock.get("https://api.paystack.co/customer", json={ "status": True }) + assert source.check_connection(logger_mock, config_mock) == (True, None) +def test_check_connection_failure(mocker, requests_mock): + source = SourcePaystack() + logger_mock, config_mock = MagicMock(), MagicMock() + requests_mock.get("https://api.paystack.co/customer", json={ "status": False, "message": "Failed" }) + + assert source.check_connection(logger_mock, config_mock) == (False, ANY) + +def test_check_connection_error(mocker, requests_mock): + source = SourcePaystack() + logger_mock, config_mock = MagicMock(), MagicMock() + requests_mock.get("https://api.paystack.co/customer", exc=requests.exceptions.ConnectTimeout) + + assert source.check_connection(logger_mock, config_mock) == (False, ANY) def test_streams(mocker): source = SourcePaystack() - config_mock = MagicMock() - streams = source.streams(config_mock) - # TODO: replace this with your streams number - expected_streams_number = 2 - assert len(streams) == expected_streams_number + streams = source.streams({"start_date": "2020-08-01", "secret_key": "sk_test_123456"}) + + assert len(streams) == 1 diff --git a/airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py index 7ecdde9868e1..5b5fc0372808 100644 --- a/airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py @@ -6,8 +6,10 @@ from unittest.mock import MagicMock import pytest -from source_paystack.source import PaystackStream +from source_paystack.streams import PaystackStream +# 1596240000 is equivalent to pendulum.parse("2020-08-01T00:00:00Z").int_timestamp +START_DATE = 1596240000 @pytest.fixture def patch_base_class(mocker): @@ -17,48 +19,53 @@ def patch_base_class(mocker): mocker.patch.object(PaystackStream, "__abstractmethods__", set()) -def test_request_params(patch_base_class): - stream = PaystackStream() - # TODO: replace this with your input parameters +def test_request_params_includes_pagination_limit(patch_base_class): + stream = PaystackStream(start_date=START_DATE) inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request parameters - expected_params = {} - assert stream.request_params(**inputs) == expected_params + params = stream.request_params(**inputs) -def test_next_page_token(patch_base_class): - stream = PaystackStream() - # TODO: replace this with your input parameters - inputs = {"response": MagicMock()} - # TODO: replace this with your expected next page token - expected_token = None - assert stream.next_page_token(**inputs) == expected_token + assert params == {"perPage": 200} +def test_request_params_for_includes_page_number_for_pagination(patch_base_class): + stream = PaystackStream(start_date=START_DATE) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": {"page": 2}} -def test_parse_response(patch_base_class): - stream = PaystackStream() - # TODO: replace this with your input parameters - inputs = {"response": MagicMock()} - # TODO: replace this with your expected parced object - expected_parsed_object = {} - assert next(stream.parse_response(**inputs)) == expected_parsed_object + params = stream.request_params(**inputs) + assert params == {"perPage": 200, "page": 2} -def test_request_headers(patch_base_class): - stream = PaystackStream() - # TODO: replace this with your input parameters - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request headers - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers +def test_next_page_token_increments_page_number(patch_base_class): + stream = PaystackStream(start_date=START_DATE) + mock_response = MagicMock() + mock_response.json.return_value = {"meta": {"page": 2, "pageCount": 4}} + inputs = {"response": mock_response} + + token = stream.next_page_token(**inputs) + + assert token == {"page": 3} + +def test_next_page_token_is_none_when_last_page_reached(patch_base_class): + stream = PaystackStream(start_date=START_DATE) + mock_response = MagicMock() + mock_response.json.return_value = {"meta": {"page": 4, "pageCount": 4}} + inputs = {"response": mock_response} + + token = stream.next_page_token(**inputs) + assert token is None -def test_http_method(patch_base_class): - stream = PaystackStream() - # TODO: replace this with your expected http request method - expected_method = "GET" - assert stream.http_method == expected_method +def test_parse_response_generates_data(patch_base_class): + stream = PaystackStream(start_date=START_DATE) + mock_response = MagicMock() + mock_response.json.return_value = {"data": [{"id": 1137850082}, {"id": 1137850097}]} + inputs = {"response": mock_response} + + parsed = stream.parse_response(**inputs) + first, second = next(parsed), next(parsed) + assert first == {"id": 1137850082} + assert second == {"id": 1137850097} @pytest.mark.parametrize( ("http_status", "should_retry"), @@ -72,12 +79,12 @@ def test_http_method(patch_base_class): def test_should_retry(patch_base_class, http_status, should_retry): response_mock = MagicMock() response_mock.status_code = http_status - stream = PaystackStream() + stream = PaystackStream(start_date=START_DATE) assert stream.should_retry(response_mock) == should_retry def test_backoff_time(patch_base_class): response_mock = MagicMock() - stream = PaystackStream() + stream = PaystackStream(start_date=START_DATE) expected_backoff_time = None assert stream.backoff_time(response_mock) == expected_backoff_time From 2ccdb0c5252dc4c988e75c4b41aa363dbbd05b84 Mon Sep 17 00:00:00 2001 From: Foluso Ogunlana Date: Tue, 19 Oct 2021 14:54:07 +0100 Subject: [PATCH 10/22] chore(67): store formatted date time in state to match type of catalog --- .../source-paystack/sample_files/state.json | 2 +- .../source-paystack/source_paystack/source.py | 3 +-- .../source-paystack/source_paystack/streams.py | 14 +++++++++----- .../unit_tests/test_incremental_streams.py | 15 +++++++-------- .../source-paystack/unit_tests/test_streams.py | 3 +-- 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/airbyte-integrations/connectors/source-paystack/sample_files/state.json b/airbyte-integrations/connectors/source-paystack/sample_files/state.json index 2747c731e448..a56485f5306c 100644 --- a/airbyte-integrations/connectors/source-paystack/sample_files/state.json +++ b/airbyte-integrations/connectors/source-paystack/sample_files/state.json @@ -1,3 +1,3 @@ { - "customers": { "createdAt": 1596240000 } + "customers": { "createdAt": "2020-08-01T00:00:00Z" } } diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/source.py b/airbyte-integrations/connectors/source-paystack/source_paystack/source.py index 9018d07bd80b..a12fee6a22c7 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/source.py +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/source.py @@ -48,8 +48,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: :param config: A Mapping of the user input configuration as defined in the connector spec. """ authenticator = TokenAuthenticator(config[AUTH_KEY_FIELD]) - start_date = pendulum.parse(config["start_date"]).int_timestamp - args = {"authenticator": authenticator, "start_date": start_date} + args = {"authenticator": authenticator, "start_date": config["start_date"]} incremental_args = {**args, "lookback_window_days": config.get("lookback_window_days")} return [ Customers(**incremental_args) diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py b/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py index 045fb1345cf5..608bb666c0f1 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py @@ -16,9 +16,9 @@ class PaystackStream(HttpStream, ABC): url_base = PAYSTACK_API_BASE_URL primary_key = "id" - def __init__(self, start_date: int, **kwargs): + def __init__(self, start_date: str, **kwargs): super().__init__(**kwargs) - self.start_date = start_date + self.start_date = pendulum.parse(start_date).int_timestamp def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: decoded_response = response.json() @@ -73,8 +73,9 @@ def get_updated_state( latest_record_created = latest_record.get(self.cursor_field) return { self.cursor_field: max( - pendulum.parse(latest_record_created).int_timestamp, - current_stream_state.get(self.cursor_field, 0), + latest_record_created, + current_stream_state.get(self.cursor_field, None), + key=lambda d: pendulum.parse(d).int_timestamp if d else 0 ) } @@ -88,7 +89,10 @@ def _get_start_date(self, stream_state) -> str: start_point = self.start_date if stream_state and self.cursor_field in stream_state: stream_record_created = stream_state[self.cursor_field] - start_point = max(start_point, stream_record_created) + start_point = max( + start_point, + pendulum.parse(stream_record_created).int_timestamp + ) if start_point and self.lookback_window_days: self.logger.info(f"Applying lookback window of {self.lookback_window_days} days to stream {self.name}") diff --git a/airbyte-integrations/connectors/source-paystack/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-paystack/unit_tests/test_incremental_streams.py index 657b9dfcbf2c..db2af442f3aa 100644 --- a/airbyte-integrations/connectors/source-paystack/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-paystack/unit_tests/test_incremental_streams.py @@ -11,8 +11,7 @@ from pytest import fixture from source_paystack.streams import IncrementalPaystackStream, Customers -# 1596240000 is equivalent to pendulum.parse("2020-08-01T00:00:00Z").int_timestamp -START_DATE = 1596240000 +START_DATE = "2020-08-01T00:00:00Z" @fixture def patch_incremental_base_class(mocker): @@ -26,17 +25,17 @@ def patch_incremental_base_class(mocker): def test_get_updated_state_uses_timestamp_of_latest_record(patch_incremental_base_class): stream = IncrementalPaystackStream(start_date=START_DATE) inputs = { - "current_stream_state": {stream.cursor_field: pendulum.parse("2021-08-01").int_timestamp}, + "current_stream_state": {stream.cursor_field: "2021-08-01"}, "latest_record": {stream.cursor_field: "2021-08-02T00:00:00Z"} } updated_state = stream.get_updated_state(**inputs) - assert updated_state == {stream.cursor_field: pendulum.parse("2021-08-02T00:00:00Z").int_timestamp} + assert updated_state == {stream.cursor_field: "2021-08-02T00:00:00Z"} def test_get_updated_state_returns_current_state_for_old_records(patch_incremental_base_class): stream = IncrementalPaystackStream(start_date=START_DATE) - current_state = {stream.cursor_field: pendulum.parse("2021-08-03").int_timestamp} + current_state = {stream.cursor_field: "2021-08-03"} inputs = { "current_stream_state": current_state, "latest_record": {stream.cursor_field: "2021-08-02T00:00:00Z"} @@ -55,14 +54,14 @@ def test_get_updated_state_uses_timestamp_of_latest_record_when_no_current_state updated_state = stream.get_updated_state(**inputs) - assert updated_state == {stream.cursor_field: pendulum.parse("2021-08-02T00:00:00Z").int_timestamp} + assert updated_state == {stream.cursor_field: "2021-08-02T00:00:00Z"} def test_request_params_includes_incremental_start_point(patch_incremental_base_class): stream = IncrementalPaystackStream(start_date=START_DATE) inputs = { "stream_slice": None, "next_page_token": {"page": 37}, - "stream_state": {stream.cursor_field: pendulum.parse("2021-09-02").int_timestamp}, + "stream_state": {stream.cursor_field: "2021-09-02"}, } params = stream.request_params(**inputs) @@ -89,7 +88,7 @@ def test_request_params_incremental_start_point_applies_lookback_window( inputs = { "stream_slice": None, "next_page_token": None, - "stream_state": {stream.cursor_field: pendulum.parse(current_state).int_timestamp}, + "stream_state": {stream.cursor_field: current_state}, } params = stream.request_params(**inputs) diff --git a/airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py index 5b5fc0372808..6c6000eb69ae 100644 --- a/airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py @@ -8,8 +8,7 @@ import pytest from source_paystack.streams import PaystackStream -# 1596240000 is equivalent to pendulum.parse("2020-08-01T00:00:00Z").int_timestamp -START_DATE = 1596240000 +START_DATE = "2020-08-01T00:00:00Z" @pytest.fixture def patch_base_class(mocker): From 67207dc92aa65908941db7a92fbaa20d04d0c62f Mon Sep 17 00:00:00 2001 From: Foluso Ogunlana Date: Tue, 19 Oct 2021 14:54:47 +0100 Subject: [PATCH 11/22] chore(67): add configuration for acceptance integration tests --- .../source-paystack/acceptance-test-config.yml | 12 +----------- .../integration_tests/abnormal_state.json | 5 ----- .../source-paystack/integration_tests/acceptance.py | 2 -- .../integration_tests/configured_catalog.json | 12 ++---------- .../integration_tests/invalid_config.json | 3 ++- .../integration_tests/sample_config.json | 3 ++- .../integration_tests/sample_state.json | 4 +--- 7 files changed, 8 insertions(+), 33 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-paystack/integration_tests/abnormal_state.json diff --git a/airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml b/airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml index a198287e8f01..444baac2148d 100644 --- a/airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml @@ -1,5 +1,3 @@ -# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) -# for more information about how to configure these tests connector_image: airbyte/source-paystack:dev tests: spec: @@ -14,17 +12,9 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.txt" -# extra_fields: no -# exact_order: no -# extra_records: yes - incremental: # TODO if your connector does not implement incremental sync, remove this block + incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-paystack/integration_tests/abnormal_state.json deleted file mode 100644 index 52b0f2c2118f..000000000000 --- a/airbyte-integrations/connectors/source-paystack/integration_tests/abnormal_state.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "todo-stream-name": { - "todo-field-name": "todo-abnormal-value" - } -} diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-paystack/integration_tests/acceptance.py index 58c194c5d137..108075487440 100644 --- a/airbyte-integrations/connectors/source-paystack/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/acceptance.py @@ -11,6 +11,4 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """ This fixture is a placeholder for external resources that acceptance test might require.""" - # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield - # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-paystack/integration_tests/configured_catalog.json index 36f0468db0d8..6f9d10061e1e 100644 --- a/airbyte-integrations/connectors/source-paystack/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/configured_catalog.json @@ -4,19 +4,11 @@ "stream": { "name": "customers", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "employees", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-paystack/integration_tests/invalid_config.json index f3732995784f..02886bf4c1dd 100644 --- a/airbyte-integrations/connectors/source-paystack/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/invalid_config.json @@ -1,3 +1,4 @@ { - "todo-wrong-field": "this should be an incomplete config file, used in standard tests" + "secret_key": "sk_live_123", + "start_date": "2020-07-01T00:00:00Z" } diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-paystack/integration_tests/sample_config.json index ecc4913b84c7..6e33d086d3dc 100644 --- a/airbyte-integrations/connectors/source-paystack/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/sample_config.json @@ -1,3 +1,4 @@ { - "fix-me": "TODO" + "secret_key": "sk_test_abc123", + "start_date": "2020-08-01T00:00:00Z" } diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-paystack/integration_tests/sample_state.json index 3587e579822d..e8f8f6a9fdea 100644 --- a/airbyte-integrations/connectors/source-paystack/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/sample_state.json @@ -1,5 +1,3 @@ { - "todo-stream-name": { - "todo-field-name": "value" - } + "customers": { "createdAt": "2020-08-01T00:00:00Z" } } From 125725b9cce1c2d3aac558b7fa16992378bcd639 Mon Sep 17 00:00:00 2001 From: Foluso Ogunlana Date: Tue, 19 Oct 2021 17:20:22 +0100 Subject: [PATCH 12/22] docs(67): update docs and summary with paystack --- docs/SUMMARY.md | 1 + docs/integrations/sources/paystack.md | 57 +++++---------------------- 2 files changed, 10 insertions(+), 48 deletions(-) diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 9c3a25a8c8fd..ff2b6317fe35 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -94,6 +94,7 @@ * [Oracle Peoplesoft](integrations/sources/oracle-peoplesoft.md) * [Oracle Siebel CRM](integrations/sources/oracle-siebel-crm.md) * [Paypal Transaction](integrations/sources/paypal-transaction.md) + * [Paystack](integrations/sources/paystack.md) * [Plaid](integrations/sources/plaid.md) * [Pipedrive](integrations/sources/pipedrive.md) * [PokéAPI](integrations/sources/pokeapi.md) diff --git a/docs/integrations/sources/paystack.md b/docs/integrations/sources/paystack.md index 0bdd580a40d1..a5790785f3d3 100644 --- a/docs/integrations/sources/paystack.md +++ b/docs/integrations/sources/paystack.md @@ -1,4 +1,4 @@ -# Stripe +# Paystack ## Overview @@ -6,51 +6,32 @@ The Paystack source supports both Full Refresh and Incremental syncs. You can ch ### Output schema -This Source is capable of syncing the following core Streams: - - +This Source is capable of syncing the following core streams: + +[Customers](https://paystack.com/docs/api/#customer-list) \(Incremental\) ### Note on Incremental Syncs -The Paystack API does not allow querying objects which were updated since the last sync. Therefore, this connector uses the `created_at` field to query for new data in your Paystack account. +The Paystack API does not allow querying objects which were updated since the last sync. Therefore, this connector uses the `createdAt` field to query for new data in your Paystack account. If your data is updated after creation, you can use the Loockback Window option when configuring the connector to always reload data from the past N days. This will allow you to pick up updates to the data. ### Data type mapping - +The [Paystack API](https://paystack.com/docs/api) is compatible with the [JSONSchema](https://json-schema.org/understanding-json-schema/reference/index.html) types that Airbyte uses internally \(`string`, `date-time`, `object`, `array`, `boolean`, `integer`, and `number`\), so no type conversions happen as part of this source. ### Features | Feature | Supported? | | :--- | :--- | - ### Performance considerations - +The Paystack connector should not run into Paystack API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. ## Getting started @@ -64,24 +45,4 @@ Visit the [Paystack dashboard settings page](https://dashboard.paystack.com/#/se Unfortunately Paystack does not yet support restricted permission levels on secret keys. This means that you will have to use the same secret key here that you use for charging customers. Use at your own risk. In the future Paystack might support restricted access levels and in that case Airbyte only requires a read-only access level key. -If you would like to test Airbyte using test data on Paystack, `sk_test_` API keys are also supported. - -## Changelog - -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | - +If you would like to test Airbyte using test data on Paystack, `sk_test_` API keys are also supported. \ No newline at end of file From f58ea83c42e899d852270190106c59f4744d6335 Mon Sep 17 00:00:00 2001 From: Foluso Ogunlana Date: Tue, 19 Oct 2021 18:51:47 +0100 Subject: [PATCH 13/22] chore(67): add essential schemas to be catalogued for new streams --- .../source_paystack/schemas/TODO.md | 25 --- .../source_paystack/schemas/customers.json | 2 +- .../source_paystack/schemas/disputes.json | 104 ++++++++++++ .../source_paystack/schemas/invoices.json | 92 +++++++++++ .../source_paystack/schemas/refunds.json | 57 +++++++ .../source_paystack/schemas/settlements.json | 40 +++++ .../schemas/subscriptions.json | 72 +++++++++ .../source_paystack/schemas/transactions.json | 151 ++++++++++++++++++ .../source_paystack/schemas/transfers.json | 43 +++++ 9 files changed, 560 insertions(+), 26 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/schemas/TODO.md create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/schemas/disputes.json create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/schemas/invoices.json create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/schemas/refunds.json create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/schemas/settlements.json create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/schemas/subscriptions.json create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transactions.json create mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transfers.json diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/TODO.md b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/TODO.md deleted file mode 100644 index cf1efadb3c9c..000000000000 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/TODO.md +++ /dev/null @@ -1,25 +0,0 @@ -# TODO: Define your stream schemas -Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). - -The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. - -The schema of a stream is the return value of `Stream.get_json_schema`. - -## Static schemas -By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. - -Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. - -## Dynamic schemas -If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). - -## Dynamically modifying static schemas -Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: -``` -def get_json_schema(self): - schema = super().get_json_schema() - schema['dynamically_determined_property'] = "property" - return schema -``` - -Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json index 7cc9020ad0a4..e833d5dbbb5f 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json @@ -41,4 +41,4 @@ "format": "date-time" } } -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/disputes.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/disputes.json new file mode 100644 index 000000000000..b35f6c217c28 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/disputes.json @@ -0,0 +1,104 @@ +{ + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "refund_amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "resolution": { + "type": ["null", "string"] + }, + "domain": { + "type": ["null", "string"] + }, + "transaction": { + "type": ["null", "object"], + "properties": {} + }, + "transaction_reference": { + "type": ["null", "string"] + }, + "category": { + "type": ["null", "string"] + }, + "customer": { + "type": ["null", "object"], + "properties": {} + }, + "bin": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + }, + "dueAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "resolvedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "evidence": { + "type": ["null", "string"] + }, + "attachments": { + "type": ["null", "string"] + }, + "note": { + "type": ["null", "string"] + }, + "history": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "status": { + "type": ["null", "string"] + }, + "by": { + "type": ["null", "string"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + } + } + } + }, + "messages": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "sender": { + "type": ["null", "string"] + }, + "body": { + "type": ["null", "string"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + } + } + } + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/invoices.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/invoices.json new file mode 100644 index 000000000000..fce30fcb5860 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/invoices.json @@ -0,0 +1,92 @@ +{ + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "domain": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "due_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "has_invoice": { + "type": ["null", "boolean"] + }, + "invoice_number": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + }, + "pdf_url": { + "type": ["null", "null"] + }, + "line_items": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + } + } + } + }, + "tax": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + } + } + } + }, + "request_code": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "paid": { + "type": ["null", "boolean"] + }, + "paid_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "metadata": { + "type": ["null", "null"] + }, + "notifications": { + "type": ["null", "array"], + "items": {} + }, + "offline_reference": { + "type": ["null", "string"] + }, + "customer": { + "type": ["null", "object"], + "properties": {} + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/refunds.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/refunds.json new file mode 100644 index 000000000000..3b6a86f7cc36 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/refunds.json @@ -0,0 +1,57 @@ +{ + "type": ["null", "object"], + "properties": { + "integration": { + "type": ["null", "integer"] + }, + "transaction": { + "type": ["null", "integer"] + }, + "dispute": { + "type": ["null", "object"], + "properties": {} + }, + "settlement": { + "type": ["null", "object"], + "properties": {} + }, + "id": { + "type": ["null", "integer"] + }, + "domain": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "string"] + }, + "refunded_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "refunded_by": { + "type": ["null", "string"] + }, + "customer_note": { + "type": ["null", "string"] + }, + "merchant_note": { + "type": ["null", "string"] + }, + "deducted_amount": { + "type": ["null", "integer"] + }, + "fully_deducted": { + "type": ["null", "boolean"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/settlements.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/settlements.json new file mode 100644 index 000000000000..10637b95c2d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/settlements.json @@ -0,0 +1,40 @@ +{ + "type": ["null", "object"], + "properties": { + "integration": { + "type": ["null", "integer"] + }, + "subaccount": { + "type": ["null", "object"], + "properties": {} + }, + "settled_by": { + "type": ["null", "object"], + "properties": {} + }, + "settlement_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "domain": { + "type": ["null", "string"] + }, + "total_amount": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/subscriptions.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/subscriptions.json new file mode 100644 index 000000000000..3d2779bd3b9b --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/subscriptions.json @@ -0,0 +1,72 @@ +{ + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "domain": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "start": { + "type": ["null", "integer"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "subscription_code": { + "type": ["null", "string"] + }, + "email_token": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "cron_expression": { + "type": ["null", "string"] + }, + "next_payment_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "open_invoice": { + "type": ["null", "object"], + "properties": {} + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "integration": { + "type": ["null", "integer"] + }, + "plan": { + "type": ["null", "object"], + "properties": {} + }, + "authorization": { + "type": ["null", "object"], + "properties": {} + }, + "customer": { + "type": ["null", "object"], + "properties": {} + }, + "invoice_limit": { + "type": ["null", "integer"] + }, + "split_code": { + "type": ["null", "null"] + }, + "payments_count": { + "type": ["null", "integer"] + }, + "most_recent_invoice": { + "type": ["null", "object"], + "properties": {} + } + } +} diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transactions.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transactions.json new file mode 100644 index 000000000000..0d9023a63347 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transactions.json @@ -0,0 +1,151 @@ +{ + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "domain": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "reference": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "message": { + "type": ["null", "string"] + }, + "gateway_response": { + "type": ["null", "string"] + }, + "paid_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "channel": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "ip_address": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"], + "properties": { + "referrer": { + "type": ["null", "string"] + } + } + }, + "log": { + "type": ["null", "object"], + "properties": { + "start_time": { + "type": ["null", "integer"] + }, + "time_spent": { + "type": ["null", "integer"] + }, + "attempts": { + "type": ["null", "integer"] + }, + "errors": { + "type": ["null", "integer"] + }, + "success": { + "type": ["null", "boolean"] + }, + "mobile": { + "type": ["null", "boolean"] + }, + "input": { + "type": ["null", "array"], + "items": {} + }, + "history": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "message": { + "type": ["null", "string"] + }, + "time": { + "type": ["null", "integer"] + } + } + } + } + } + }, + "fees": { + "type": ["null", "null"] + }, + "fees_split": { + "type": ["null", "null"] + }, + "customer": { + "type": ["null", "object"], + "properties": {} + }, + "authorization": { + "type": ["null", "object"], + "properties": {} + } + }, + "plan": { + "type": ["null", "object"], + "properties": {} + }, + "split": { + "type": ["null", "object"], + "properties": {} + }, + "subaccount": { + "type": ["null", "object"], + "properties": {} + }, + "order_id": { + "type": ["null", "null"] + }, + "paidAt": { + "type": ["null", "null"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "requested_amount": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "object"], + "properties": { + "source": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "identifier": { + "type": ["null", "null"] + }, + "entry_point": { + "type": ["null", "string"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transfers.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transfers.json new file mode 100644 index 000000000000..4607e522cd40 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transfers.json @@ -0,0 +1,43 @@ +{ + "type": ["null", "object"], + "properties": { + "integration": { + "type": ["null", "integer"] + }, + "domain": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "source": { + "type": ["null", "string"] + }, + "reason": { + "type": ["null", "string"] + }, + "recipient": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "string"] + }, + "transfer_code": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + } + } +} From 0296fed4c72285d1a173816f5c02f9e31acd068f Mon Sep 17 00:00:00 2001 From: Foluso Ogunlana Date: Wed, 20 Oct 2021 07:22:01 +0100 Subject: [PATCH 14/22] feat(67): add support for critical streams - transactions subscriptions transfers refunds settlements --- .../connectors/source-paystack/ BOOTSTRAP.md | 11 + .../connectors/source-paystack/BOOTSTRAP.md | 21 + .../acceptance-test-config.yml | 1 + .../integration_tests/catalog.json | 39 - .../integration_tests/configured_catalog.json | 80 +- .../integration_tests/sample_state.json | 9 +- .../sample_files/configured_catalog.json | 817 ++++++++++++++++-- .../source-paystack/sample_files/state.json | 9 +- .../source_paystack/constants.py | 3 +- .../source_paystack/schemas/customers.json | 2 +- .../source_paystack/schemas/invoices.json | 18 +- .../source_paystack/schemas/settlements.json | 15 +- .../schemas/subscriptions.json | 3 +- .../source_paystack/schemas/transactions.json | 85 +- .../source_paystack/schemas/transfers.json | 60 +- .../source-paystack/source_paystack/source.py | 20 +- .../source_paystack/streams.py | 83 +- .../source-paystack/unit_tests/test_source.py | 2 +- .../unit_tests/test_streams.py | 10 + docs/integrations/sources/paystack.md | 7 + 20 files changed, 1138 insertions(+), 157 deletions(-) create mode 100644 airbyte-integrations/connectors/source-paystack/ BOOTSTRAP.md create mode 100644 airbyte-integrations/connectors/source-paystack/BOOTSTRAP.md delete mode 100644 airbyte-integrations/connectors/source-paystack/integration_tests/catalog.json diff --git a/airbyte-integrations/connectors/source-paystack/ BOOTSTRAP.md b/airbyte-integrations/connectors/source-paystack/ BOOTSTRAP.md new file mode 100644 index 000000000000..d3d74777cacc --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/ BOOTSTRAP.md @@ -0,0 +1,11 @@ +# Cart + +paystack.com is a straightforward CRUD REST API. Connector is implemented with [Airbyte CDK](https://docs.airbyte.io/connector-development/cdk-python). + +It consists of some REST resources like customers, transactions, transfers, subscriptions, etc. each of which have a list endpoint with a "from" filter that can be used to perform incremental syncs. + +Auth uses an API token which can be found in the Paystack dashboard. +Pagination uses a cursor pagination strategy. + +See the links below for information about specific streams and some nuances about the connector: +- [Paystack connector documentation](https://docs.airbyte.io/integrations/sources/paystack) diff --git a/airbyte-integrations/connectors/source-paystack/BOOTSTRAP.md b/airbyte-integrations/connectors/source-paystack/BOOTSTRAP.md new file mode 100644 index 000000000000..94fffb8f7253 --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/BOOTSTRAP.md @@ -0,0 +1,21 @@ +# Paystack + +paystack.com is a Payment Gateway and its REST API is similar to Stripe's. This Paystack API connector is implemented with [Airbyte CDK](https://docs.airbyte.io/connector-development/cdk-python). + +The Paystack API has resources including (not exhaustive) +- Customers +- Transactions - Payments and payment attempts +- Subscriptions - Recurring payments +- Invoices - Requests for payment +- Settlements - Transfers from the Paystack Gateway account to the merchant (account instance owner) bank account +- Refunds - Reversed payments + +The Paystack API can be used to charge customers, and to perform CRUD operations on any of the above resources. For Airbyte only the "R" - read operations are needed, however Paystack currently supports a single secret key which can do all CRUD operations. + +## Notes & Quirks +- Pagination uses the query parameters "page" (starting at 1) and "perPage". +- The standard cursor field is "createdAt" on all responses, except the "Invoices" stream which uses "created_at". It's likely the interface for this resource is either outdated or failed to be backward compatible (some other resources have both fields and some have only "createdAt"). + +## Useful links below +- [Paystack connector documentation](https://docs.airbyte.io/integrations/sources/paystack) - Information about specific streams and some nuances about the connector +- [Paystack dashboard](https://dashboard.paystack.com/#/settings/developer) - To grab your API token \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml b/airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml index 444baac2148d..3c59848ec0aa 100644 --- a/airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml @@ -12,6 +12,7 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: ['disputes', 'transfers', 'settlements', 'invoices'] incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/catalog.json b/airbyte-integrations/connectors/source-paystack/integration_tests/catalog.json deleted file mode 100644 index 6799946a6851..000000000000 --- a/airbyte-integrations/connectors/source-paystack/integration_tests/catalog.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "streams": [ - { - "name": "TODO fix this file", - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": "column1", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "column1": { - "type": "string" - }, - "column2": { - "type": "number" - } - } - } - }, - { - "name": "table1", - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "column1": { - "type": "string" - }, - "column2": { - "type": "number" - } - } - } - } - ] -} diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-paystack/integration_tests/configured_catalog.json index 6f9d10061e1e..12a2cc468b91 100644 --- a/airbyte-integrations/connectors/source-paystack/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/configured_catalog.json @@ -1,14 +1,84 @@ { "streams": [ { + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", "stream": { "name": "customers", - "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "source_defined_cursor": true, + "json_schema": {} + } + }, + { + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "stream": { + "name": "disputes", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "json_schema": {} + } + }, + { + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "stream": { + "name": "invoices", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "json_schema": {} + } + }, + { + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "stream": { + "name": "refunds", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "json_schema": {} + } + }, + { + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "stream": { + "name": "settlements", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "json_schema": {} + } + }, + { + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "stream": { + "name": "subscriptions", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "json_schema": {} + } + }, + { + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "stream": { + "name": "transactions", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "json_schema": {} + } + }, + { + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "stream": { + "name": "transfers", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "json_schema": {} + } } ] } diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-paystack/integration_tests/sample_state.json index e8f8f6a9fdea..2b7a6496036c 100644 --- a/airbyte-integrations/connectors/source-paystack/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/sample_state.json @@ -1,3 +1,10 @@ { - "customers": { "createdAt": "2020-08-01T00:00:00Z" } + "customers": { "createdAt": "2019-08-01T00:00:00Z" }, + "disputes": { "createdAt": "2019-08-01T00:00:00Z" }, + "invoices": { "created_at": "2019-08-01T00:00:00Z" }, + "refunds": { "createdAt": "2019-08-01T00:00:00Z" }, + "settlements": { "createdAt": "2019-08-01T00:00:00Z" }, + "subscriptions": { "createdAt": "2019-08-01T00:00:00Z" }, + "transactions": { "createdAt": "2019-08-01T00:00:00Z" }, + "transfers": { "createdAt": "2019-08-01T00:00:00Z" } } diff --git a/airbyte-integrations/connectors/source-paystack/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-paystack/sample_files/configured_catalog.json index 51e670ae21f7..1cec82dbc835 100644 --- a/airbyte-integrations/connectors/source-paystack/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-paystack/sample_files/configured_catalog.json @@ -1,57 +1,764 @@ { - "streams": [ - { - "stream": { - "name": "customers", - "json_schema": { - "type": ["null", "object"], - "properties": { - "integration": { - "type": ["null", "integer"] - }, - "first_name": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "metadata": { + "streams": [ + { + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "stream": { + "name": "customers", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "json_schema": { + "type": ["null", "object"], + "properties": { + "integration": { + "type": ["null", "integer"] + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "domain": { + "type": ["null", "string"] + }, + "customer_code": { + "type": ["null", "string"] + }, + "risk_action": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + } + } + } + } + }, + { + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "stream": { + "name": "disputes", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "json_schema": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "refund_amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "resolution": { + "type": ["null", "string"] + }, + "domain": { + "type": ["null", "string"] + }, + "transaction": { + "type": ["null", "object"], + "properties": {} + }, + "transaction_reference": { + "type": ["null", "string"] + }, + "category": { + "type": ["null", "string"] + }, + "customer": { + "type": ["null", "object"], + "properties": {} + }, + "bin": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + }, + "dueAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "resolvedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "evidence": { + "type": ["null", "string"] + }, + "attachments": { + "type": ["null", "string"] + }, + "note": { + "type": ["null", "string"] + }, + "history": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "status": { + "type": ["null", "string"] + }, + "by": { + "type": ["null", "string"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + } + } + } + }, + "messages": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "sender": { + "type": ["null", "string"] + }, + "body": { + "type": ["null", "string"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + } + } + } + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + } + } + } + } + }, + { + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "stream": { + "name": "invoices", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "json_schema": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "integration": { + "type": ["null", "integer"] + }, + "domain": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "due_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "has_invoice": { + "type": ["null", "boolean"] + }, + "invoice_number": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + }, + "pdf_url": { + "type": ["null", "null"] + }, + "line_items": { + "type": ["null", "array"], + "items": { "type": ["null", "object"], - "properties": {} - }, - "domain": { - "type": ["null", "string"] - }, - "customer_code": { - "type": ["null", "string"] - }, - "risk_action": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "updatedAt": { - "type": ["null", "string"], - "format": "date-time" - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": true - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} \ No newline at end of file + "properties": { + "name": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "quantity": { + "type": ["null", "integer"] + } + } + } + }, + "tax": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + } + } + } + }, + "request_code": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "paid": { + "type": ["null", "boolean"] + }, + "paid_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "metadata": { + "type": ["null", "null"] + }, + "notifications": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "sent_at": { + "type": ["null", "string"] + }, + "channel": { + "type": ["null", "string"] + } + } + } + }, + "offline_reference": { + "type": ["null", "string"] + }, + "customer": { + "type": ["null", "object"], + "properties": {} + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + } + } + }, + { + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "stream": { + "name": "refunds", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "json_schema": { + "type": ["null", "object"], + "properties": { + "integration": { + "type": ["null", "integer"] + }, + "transaction": { + "type": ["null", "integer"] + }, + "dispute": { + "type": ["null", "object"], + "properties": {} + }, + "settlement": { + "type": ["null", "object"], + "properties": {} + }, + "id": { + "type": ["null", "integer"] + }, + "domain": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "string"] + }, + "refunded_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "refunded_by": { + "type": ["null", "string"] + }, + "customer_note": { + "type": ["null", "string"] + }, + "merchant_note": { + "type": ["null", "string"] + }, + "deducted_amount": { + "type": ["null", "integer"] + }, + "fully_deducted": { + "type": ["null", "boolean"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + } + } + } + } + }, + { + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "stream": { + "name": "settlements", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "json_schema": { + "type": ["null", "object"], + "properties": { + "integration": { + "type": ["null", "integer"] + }, + "subaccount": { + "type": ["null", "object"], + "properties": {} + }, + "settled_by": { + "type": ["null", "string"] + }, + "settlement_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "domain": { + "type": ["null", "string"] + }, + "total_amount": { + "type": ["null", "integer"] + }, + "total_fees": { + "type": ["null", "integer"] + }, + "total_processed": { + "type": ["null", "integer"] + }, + "deductions": { + "type": ["null", "integer"] + }, + "effective_amount": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + } + } + } + } + }, + { + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "stream": { + "name": "subscriptions", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "json_schema": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "domain": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "start": { + "type": ["null", "integer"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "subscription_code": { + "type": ["null", "string"] + }, + "email_token": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "cron_expression": { + "type": ["null", "string"] + }, + "next_payment_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "open_invoice": { + "type": ["null", "string"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "integration": { + "type": ["null", "integer"] + }, + "plan": { + "type": ["null", "object"], + "properties": {} + }, + "authorization": { + "type": ["null", "object"], + "properties": {} + }, + "customer": { + "type": ["null", "object"], + "properties": {} + }, + "invoice_limit": { + "type": ["null", "integer"] + }, + "split_code": { + "type": ["null", "null"] + }, + "payments_count": { + "type": ["null", "integer"] + }, + "most_recent_invoice": { + "type": ["null", "object"], + "properties": {} + } + } + } + } + }, + { + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "stream": { + "name": "transactions", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "json_schema": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "domain": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "reference": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "message": { + "type": ["null", "string"] + }, + "gateway_response": { + "type": ["null", "string"] + }, + "paid_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "channel": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "ip_address": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"], + "properties": { + "referrer": { + "type": ["null", "string"] + } + } + }, + "log": { + "type": ["null", "object"], + "properties": { + "start_time": { + "type": ["null", "integer"] + }, + "time_spent": { + "type": ["null", "integer"] + }, + "attempts": { + "type": ["null", "integer"] + }, + "errors": { + "type": ["null", "integer"] + }, + "success": { + "type": ["null", "boolean"] + }, + "mobile": { + "type": ["null", "boolean"] + }, + "input": { + "type": ["null", "array"], + "items": {} + }, + "history": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "message": { + "type": ["null", "string"] + }, + "time": { + "type": ["null", "integer"] + } + } + } + } + } + }, + "fees": { + "type": ["null", "integer"] + }, + "fees_split": { + "type": ["null", "null"] + }, + "customer": { + "type": ["null", "object"], + "properties": {} + }, + "authorization": { + "type": ["null", "object"], + "properties": {} + }, + "plan": { + "type": ["null", "object"], + "properties": {} + }, + "split": { + "type": ["null", "object"], + "properties": {} + }, + "subaccount": { + "type": ["null", "object"], + "properties": {} + }, + "order_id": { + "type": ["null", "integer"] + }, + "paidAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "requested_amount": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "object"], + "properties": { + "source": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "identifier": { + "type": ["null", "null"] + }, + "entry_point": { + "type": ["null", "string"] + } + } + } + } + } + } + }, + { + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "stream": { + "name": "transfers", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "json_schema": { + "type": ["null", "object"], + "properties": { + "integration": { + "type": ["null", "integer"] + }, + "domain": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "source": { + "type": ["null", "string"] + }, + "reason": { + "type": ["null", "string"] + }, + "recipient": { + "type": ["null", "object"], + "properties": { + "domain": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "details": { + "type": ["null", "object"], + "properties": { + "account_number": { + "type": ["null", "string"] + }, + "account_name": { + "type": ["null", "null"] + }, + "bank_code": { + "type": ["null", "string"] + }, + "bank_name": { + "type": ["null", "string"] + } + } + }, + "description": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "recipient_code": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer"] + }, + "integration": { + "type": ["null", "integer"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "status": { + "type": ["null", "string"] + }, + "transfer_code": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + } + } + } + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-paystack/sample_files/state.json b/airbyte-integrations/connectors/source-paystack/sample_files/state.json index a56485f5306c..50fa6cde6361 100644 --- a/airbyte-integrations/connectors/source-paystack/sample_files/state.json +++ b/airbyte-integrations/connectors/source-paystack/sample_files/state.json @@ -1,3 +1,10 @@ { - "customers": { "createdAt": "2020-08-01T00:00:00Z" } + "customers": { "createdAt": "2019-08-01T00:00:00Z" }, + "disputes": { "createdAt": "2019-08-01T00:00:00Z" }, + "invoices": { "created_at": "2019-08-01T00:00:00Z" }, + "refunds": { "createdAt": "2019-08-01T00:00:00Z" }, + "settlements": { "createdAt": "2019-08-01T00:00:00Z" }, + "subscriptions": { "createdAt": "2019-08-01T00:00:00Z" }, + "transactions": { "createdAt": "2019-08-01T00:00:00Z" }, + "transfers": { "createdAt": "2019-08-01T00:00:00Z" } } diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/constants.py b/airbyte-integrations/connectors/source-paystack/source_paystack/constants.py index f3412c1210a9..08be1073ec5c 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/constants.py +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/constants.py @@ -1,3 +1,4 @@ PAYSTACK_API_BASE_URL = "https://api.paystack.co/" AUTH_KEY_FIELD = "secret_key" -PAYSTACK_CREATED_AT = "createdAt" \ No newline at end of file +PAYSTACK_CREATED_AT = "createdAt" +PAYSTACK_CREATED_AT_MISC = "created_at" diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json index e833d5dbbb5f..7cc9020ad0a4 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json @@ -41,4 +41,4 @@ "format": "date-time" } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/invoices.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/invoices.json index fce30fcb5860..d8ef0d8647ef 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/invoices.json +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/invoices.json @@ -4,6 +4,9 @@ "id": { "type": ["null", "integer"] }, + "integration": { + "type": ["null", "integer"] + }, "domain": { "type": ["null", "string"] }, @@ -39,6 +42,9 @@ }, "amount": { "type": ["null", "integer"] + }, + "quantity": { + "type": ["null", "integer"] } } } @@ -75,7 +81,17 @@ }, "notifications": { "type": ["null", "array"], - "items": {} + "items": { + "type": ["null", "object"], + "properties": { + "sent_at": { + "type": ["null", "string"] + }, + "channel": { + "type": ["null", "string"] + } + } + } }, "offline_reference": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/settlements.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/settlements.json index 10637b95c2d8..e03415a022e6 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/settlements.json +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/settlements.json @@ -9,8 +9,7 @@ "properties": {} }, "settled_by": { - "type": ["null", "object"], - "properties": {} + "type": ["null", "string"] }, "settlement_date": { "type": ["null", "string"], @@ -22,6 +21,18 @@ "total_amount": { "type": ["null", "integer"] }, + "total_fees": { + "type": ["null", "integer"] + }, + "total_processed": { + "type": ["null", "integer"] + }, + "deductions": { + "type": ["null", "integer"] + }, + "effective_amount": { + "type": ["null", "integer"] + }, "status": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/subscriptions.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/subscriptions.json index 3d2779bd3b9b..190a9e3647f9 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/subscriptions.json +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/subscriptions.json @@ -33,8 +33,7 @@ "format": "date-time" }, "open_invoice": { - "type": ["null", "object"], - "properties": {} + "type": ["null", "string"] }, "createdAt": { "type": ["null", "string"], diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transactions.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transactions.json index 0d9023a63347..8372a2df86d4 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transactions.json +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transactions.json @@ -92,7 +92,7 @@ } }, "fees": { - "type": ["null", "null"] + "type": ["null", "integer"] }, "fees_split": { "type": ["null", "null"] @@ -104,47 +104,48 @@ "authorization": { "type": ["null", "object"], "properties": {} - } - }, - "plan": { - "type": ["null", "object"], - "properties": {} - }, - "split": { - "type": ["null", "object"], - "properties": {} - }, - "subaccount": { - "type": ["null", "object"], - "properties": {} - }, - "order_id": { - "type": ["null", "null"] - }, - "paidAt": { - "type": ["null", "null"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "requested_amount": { - "type": ["null", "integer"] - }, - "source": { - "type": ["null", "object"], - "properties": { - "source": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "identifier": { - "type": ["null", "null"] - }, - "entry_point": { - "type": ["null", "string"] + }, + "plan": { + "type": ["null", "object"], + "properties": {} + }, + "split": { + "type": ["null", "object"], + "properties": {} + }, + "subaccount": { + "type": ["null", "object"], + "properties": {} + }, + "order_id": { + "type": ["null", "integer"] + }, + "paidAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "requested_amount": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "object"], + "properties": { + "source": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "identifier": { + "type": ["null", "null"] + }, + "entry_point": { + "type": ["null", "string"] + } } } } diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transfers.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transfers.json index 4607e522cd40..fe5658d4abfc 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transfers.json +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transfers.json @@ -20,7 +20,65 @@ "type": ["null", "string"] }, "recipient": { - "type": ["null", "integer"] + "type": ["null", "object"], + "properties": { + "domain": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "details": { + "type": ["null", "object"], + "properties": { + "account_number": { + "type": ["null", "string"] + }, + "account_name": { + "type": ["null", "null"] + }, + "bank_code": { + "type": ["null", "string"] + }, + "bank_name": { + "type": ["null", "string"] + } + } + }, + "description": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "recipient_code": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer"] + }, + "integration": { + "type": ["null", "integer"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + } + } }, "status": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/source.py b/airbyte-integrations/connectors/source-paystack/source_paystack/source.py index a12fee6a22c7..cecda8b07567 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/source.py +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/source.py @@ -9,7 +9,16 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator from source_paystack.constants import PAYSTACK_API_BASE_URL, AUTH_KEY_FIELD -from source_paystack.streams import Customers +from source_paystack.streams import ( + Customers, + Disputes, + Invoices, + Refunds, + Settlements, + Subscriptions, + Transactions, + Transfers +) class SourcePaystack(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: @@ -51,6 +60,13 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: args = {"authenticator": authenticator, "start_date": config["start_date"]} incremental_args = {**args, "lookback_window_days": config.get("lookback_window_days")} return [ - Customers(**incremental_args) + Customers(**incremental_args), + Disputes(**incremental_args), + Invoices(**incremental_args), + Refunds(**incremental_args), + Settlements(**incremental_args), + Subscriptions(**incremental_args), + Transactions(**incremental_args), + Transfers(**incremental_args) ] diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py b/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py index 608bb666c0f1..41d2df2991e5 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py @@ -9,7 +9,7 @@ import requests import pendulum from airbyte_cdk.sources.streams.http import HttpStream -from source_paystack.constants import PAYSTACK_API_BASE_URL, PAYSTACK_CREATED_AT +from source_paystack.constants import PAYSTACK_API_BASE_URL, PAYSTACK_CREATED_AT, PAYSTACK_CREATED_AT_MISC class PaystackStream(HttpStream, ABC): @@ -24,7 +24,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, decoded_response = response.json() page, pageCount = decoded_response['meta']['page'], response.json()['meta']['pageCount'] - if page != pageCount: + if page < pageCount: return { "page": page + 1 } def request_params( @@ -109,4 +109,81 @@ class Customers(IncrementalPaystackStream): cursor_field = PAYSTACK_CREATED_AT def path(self, **kwargs) -> str: - return "customer" \ No newline at end of file + return "customer" + + +class Disputes(IncrementalPaystackStream): + """ + API docs: https://paystack.com/docs/api/#dispute-list + """ + + cursor_field = PAYSTACK_CREATED_AT + + def path(self, **kwargs) -> str: + return "dispute" + + +class Invoices(IncrementalPaystackStream): + """ + API docs: https://paystack.com/docs/api/#invoice-list + """ + + cursor_field = PAYSTACK_CREATED_AT_MISC + + def path(self, **kwargs) -> str: + return "paymentrequest" + + +class Refunds(IncrementalPaystackStream): + """ + API docs: https://paystack.com/docs/api/#refund-list + """ + + cursor_field = PAYSTACK_CREATED_AT + + def path(self, **kwargs) -> str: + return "refund" + + +class Settlements(IncrementalPaystackStream): + """ + API docs: https://paystack.com/docs/api/#settlement + """ + + cursor_field = PAYSTACK_CREATED_AT + + def path(self, **kwargs) -> str: + return "settlement" + + +class Subscriptions(IncrementalPaystackStream): + """ + API docs: https://paystack.com/docs/api/#subscription-list + """ + + cursor_field = PAYSTACK_CREATED_AT + + def path(self, **kwargs) -> str: + return "subscription" + + +class Transactions(IncrementalPaystackStream): + """ + API docs: https://paystack.com/docs/api/#transaction-list + """ + + cursor_field = PAYSTACK_CREATED_AT + + def path(self, **kwargs) -> str: + return "transaction" + + +class Transfers(IncrementalPaystackStream): + """ + API docs: https://paystack.com/docs/api/#transfer-list + """ + + cursor_field = PAYSTACK_CREATED_AT + + def path(self, **kwargs) -> str: + return "transfer" diff --git a/airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py b/airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py index 9964fa0708fc..0f12498f4b80 100644 --- a/airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py @@ -33,4 +33,4 @@ def test_streams(mocker): source = SourcePaystack() streams = source.streams({"start_date": "2020-08-01", "secret_key": "sk_test_123456"}) - assert len(streams) == 1 + assert len(streams) == 8 diff --git a/airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py index 6c6000eb69ae..b999d43e4fcc 100644 --- a/airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py @@ -54,6 +54,16 @@ def test_next_page_token_is_none_when_last_page_reached(patch_base_class): assert token is None +def test_next_page_token_is_none_when_no_pages_exist(patch_base_class): + stream = PaystackStream(start_date=START_DATE) + mock_response = MagicMock() + mock_response.json.return_value = {"meta": {"page": 1, "pageCount": 0}} + inputs = {"response": mock_response} + + token = stream.next_page_token(**inputs) + + assert token is None + def test_parse_response_generates_data(patch_base_class): stream = PaystackStream(start_date=START_DATE) mock_response = MagicMock() diff --git a/docs/integrations/sources/paystack.md b/docs/integrations/sources/paystack.md index a5790785f3d3..5194c7d5e49c 100644 --- a/docs/integrations/sources/paystack.md +++ b/docs/integrations/sources/paystack.md @@ -9,6 +9,13 @@ The Paystack source supports both Full Refresh and Incremental syncs. You can ch This Source is capable of syncing the following core streams: [Customers](https://paystack.com/docs/api/#customer-list) \(Incremental\) +[Disputes](https://paystack.com/docs/api/#dispute-list) \(Incremental\) +[Invoices](https://paystack.com/docs/api/#invoice-list) \(Incremental\) +[Refunds](https://paystack.com/docs/api/#refund-list) \(Incremental\) +[Settlements](https://paystack.com/docs/api/#settlement) \(Incremental\) +[Subscriptions](https://paystack.com/docs/api/#subscription-list) \(Incremental\) +[Transactions](https://paystack.com/docs/api/#transaction-list) \(Incremental\) +[Transfers](https://paystack.com/docs/api/#transfer-list) \(Incremental\) ### Note on Incremental Syncs From ec21bc8c3d8cf17acce19f0f7f86a4a3ed4cb237 Mon Sep 17 00:00:00 2001 From: Foluso Ogunlana Date: Wed, 20 Oct 2021 15:14:36 +0100 Subject: [PATCH 15/22] docs(67): update image and bootstrap --- .../init/src/main/resources/icons/paystack.svg | 2 +- .../connectors/source-paystack/ BOOTSTRAP.md | 11 ----------- 2 files changed, 1 insertion(+), 12 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-paystack/ BOOTSTRAP.md diff --git a/airbyte-config/init/src/main/resources/icons/paystack.svg b/airbyte-config/init/src/main/resources/icons/paystack.svg index 6cf03a4a71d1..d9374ee44d87 100644 --- a/airbyte-config/init/src/main/resources/icons/paystack.svg +++ b/airbyte-config/init/src/main/resources/icons/paystack.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/airbyte-integrations/connectors/source-paystack/ BOOTSTRAP.md b/airbyte-integrations/connectors/source-paystack/ BOOTSTRAP.md deleted file mode 100644 index d3d74777cacc..000000000000 --- a/airbyte-integrations/connectors/source-paystack/ BOOTSTRAP.md +++ /dev/null @@ -1,11 +0,0 @@ -# Cart - -paystack.com is a straightforward CRUD REST API. Connector is implemented with [Airbyte CDK](https://docs.airbyte.io/connector-development/cdk-python). - -It consists of some REST resources like customers, transactions, transfers, subscriptions, etc. each of which have a list endpoint with a "from" filter that can be used to perform incremental syncs. - -Auth uses an API token which can be found in the Paystack dashboard. -Pagination uses a cursor pagination strategy. - -See the links below for information about specific streams and some nuances about the connector: -- [Paystack connector documentation](https://docs.airbyte.io/integrations/sources/paystack) From fb30fff19159e7b9d5b0f946edf406e2f47c265b Mon Sep 17 00:00:00 2001 From: Foluso Ogunlana Date: Wed, 20 Oct 2021 15:18:22 +0100 Subject: [PATCH 16/22] chore(67): update builds.md to include paystack badge --- airbyte-integrations/builds.md | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index fb11a5ba0a6e..e3d95f65f784 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -55,6 +55,7 @@ | MySQL | [![source-mysql](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-mysql%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-mysql) | | Oracle DB | [![source-oracle](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-oracle%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-oracle) | | Paypal Transaction | [![paypal-transaction](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-paypal-transaction%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-paypal-transaction) | +| Paystack | [![source-paystack](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-paystack%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-paystack) | | Pipedrive | [![source-pipedrive](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-pipedrive%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-pipedrive) | | Plaid | [![source-plaid](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-plaid%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-plaid) | | Postgres | [![source-postgres](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-postgres%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-postgres) | From 129f4555b1496bfc99cfd89a6f2613481675159d Mon Sep 17 00:00:00 2001 From: Foluso Ogunlana Date: Wed, 20 Oct 2021 15:23:56 +0100 Subject: [PATCH 17/22] docs(67): add changelog and source definition JSON file --- .../193bdcb8-1dd9-48d1-aade-91cadfd74f9b.json | 8 ++++++++ docs/integrations/sources/paystack.md | 9 ++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/193bdcb8-1dd9-48d1-aade-91cadfd74f9b.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/193bdcb8-1dd9-48d1-aade-91cadfd74f9b.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/193bdcb8-1dd9-48d1-aade-91cadfd74f9b.json new file mode 100644 index 000000000000..eb18cc75368f --- /dev/null +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/193bdcb8-1dd9-48d1-aade-91cadfd74f9b.json @@ -0,0 +1,8 @@ +{ + "sourceDefinitionId": "193bdcb8-1dd9-48d1-aade-91cadfd74f9b", + "name": "Paystack", + "dockerRepository": "airbyte/source-paystack", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://docs.airbyte.io/integrations/sources/paystack" + } + \ No newline at end of file diff --git a/docs/integrations/sources/paystack.md b/docs/integrations/sources/paystack.md index 5194c7d5e49c..802c7148075b 100644 --- a/docs/integrations/sources/paystack.md +++ b/docs/integrations/sources/paystack.md @@ -52,4 +52,11 @@ Visit the [Paystack dashboard settings page](https://dashboard.paystack.com/#/se Unfortunately Paystack does not yet support restricted permission levels on secret keys. This means that you will have to use the same secret key here that you use for charging customers. Use at your own risk. In the future Paystack might support restricted access levels and in that case Airbyte only requires a read-only access level key. -If you would like to test Airbyte using test data on Paystack, `sk_test_` API keys are also supported. \ No newline at end of file +If you would like to test Airbyte using test data on Paystack, `sk_test_` API keys are also supported. + + +## Changelog + +| Version | Date | Pull Request | Subject | +| :--- | :--- | :--- | :--- | +| 0.1.0 | 2021-10-20 | [7214](https://github.com/airbytehq/airbyte/pull/7214) | Add Paystack source connector | \ No newline at end of file From 180969ad669b487d6e5b4c21d8fad79fe02347f2 Mon Sep 17 00:00:00 2001 From: Foluso Ogunlana Date: Wed, 20 Oct 2021 15:25:09 +0100 Subject: [PATCH 18/22] docs(67): add paystack to integrations readme --- docs/integrations/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrations/README.md b/docs/integrations/README.md index 0ce0f7212f37..470cba5c25ce 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -77,6 +77,7 @@ Airbyte uses a grading system for connectors to help users understand what to ex | [Oracle PeopleSoft](sources/oracle-peoplesoft.md) | Beta | | [Oracle Siebel CRM](sources/oracle-siebel-crm.md) | Beta | | [PayPal Transaction](sources/paypal-transaction.md) | Beta | +| [Paystack](sources/paystack.md) | Alpha | | [Pipedrive](sources/pipedrive.md) | Alpha | | [Plaid](sources/plaid.md) | Alpha | | [PokéAPI](sources/pokeapi.md) | Beta | From 99c6355757775eff4d023f19273b07447a44250a Mon Sep 17 00:00:00 2001 From: Foluso Ogunlana Date: Sat, 23 Oct 2021 13:04:46 +0100 Subject: [PATCH 19/22] chore(67): update check_connection to airbyte standard --- .../source-paystack/source_paystack/source.py | 31 +++++++------------ .../source-paystack/unit_tests/test_source.py | 22 ++++++++++--- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/source.py b/airbyte-integrations/connectors/source-paystack/source_paystack/source.py index cecda8b07567..0dac4d8b0bb6 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/source.py +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/source.py @@ -3,8 +3,8 @@ # from typing import Any, List, Mapping, Tuple -import requests import pendulum +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator @@ -30,25 +30,18 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ try: - response = requests.get( - f"{PAYSTACK_API_BASE_URL}customer?page=1&perPage=1", - headers={"Authorization": f"Bearer {config[AUTH_KEY_FIELD]}"}, - verify=True - ) - response.raise_for_status() - except Exception as e: - msg = e - if ( - isinstance(e, requests.exceptions.HTTPError) - and e.response.status_code == 401 - ): - msg = 'Connection to Paystack was not authorized. Please check that your secret key is correct.' - return False, msg + authenticator = TokenAuthenticator(token=config[AUTH_KEY_FIELD]) + stream = Customers(authenticator=authenticator, start_date=config["start_date"]) + records = stream.read_records(sync_mode=SyncMode.full_refresh) + next(records) + return True, None + + except StopIteration: + # there are no records, but connection was fine + return True, None - response_json = response.json() - if response_json['status'] == False: - return False, 'Connection test failed due to error: ' + response_json['message'] - return True, None + except Exception as e: + return False, repr(e) def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ diff --git a/airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py b/airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py index 0f12498f4b80..f4936918d1d1 100644 --- a/airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py @@ -4,14 +4,28 @@ from unittest.mock import MagicMock, ANY +import pytest import requests from source_paystack.source import SourcePaystack - -def test_check_connection_success(mocker, requests_mock): +@pytest.mark.parametrize( + ("items",), + [ + ([{ "createdAt": "2022-07-01T00:00:00Z", "id": 12345 }],), # single customer + ([],) # no customers + ], +) +def test_check_connection_success(mocker, requests_mock, items): source = SourcePaystack() - logger_mock, config_mock = MagicMock(), MagicMock() - requests_mock.get("https://api.paystack.co/customer", json={ "status": True }) + logger_mock = MagicMock() + config_mock = { + "start_date": "2020-07-01T00:00:00Z", + "secret_key": "sk_test_abc123" + } + requests_mock.get("https://api.paystack.co/customer", json={ + "data": items, + "meta": {"page": 1, "pageCount": 0}, + }) assert source.check_connection(logger_mock, config_mock) == (True, None) From 5f8947b5eb226696690c671209822ecad151ad88 Mon Sep 17 00:00:00 2001 From: Foluso Ogunlana Date: Sat, 23 Oct 2021 13:08:57 +0100 Subject: [PATCH 20/22] refactor to simplify streams and remove constants file --- .../source_paystack/constants.py | 4 ---- .../source-paystack/source_paystack/source.py | 5 ++--- .../source_paystack/streams.py | 22 +++++++++---------- 3 files changed, 13 insertions(+), 18 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-paystack/source_paystack/constants.py diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/constants.py b/airbyte-integrations/connectors/source-paystack/source_paystack/constants.py deleted file mode 100644 index 08be1073ec5c..000000000000 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/constants.py +++ /dev/null @@ -1,4 +0,0 @@ -PAYSTACK_API_BASE_URL = "https://api.paystack.co/" -AUTH_KEY_FIELD = "secret_key" -PAYSTACK_CREATED_AT = "createdAt" -PAYSTACK_CREATED_AT_MISC = "created_at" diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/source.py b/airbyte-integrations/connectors/source-paystack/source_paystack/source.py index 0dac4d8b0bb6..a2127ec2ccd2 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/source.py +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/source.py @@ -8,7 +8,6 @@ from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -from source_paystack.constants import PAYSTACK_API_BASE_URL, AUTH_KEY_FIELD from source_paystack.streams import ( Customers, Disputes, @@ -30,7 +29,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ try: - authenticator = TokenAuthenticator(token=config[AUTH_KEY_FIELD]) + authenticator = TokenAuthenticator(token=config["secret_key"]) stream = Customers(authenticator=authenticator, start_date=config["start_date"]) records = stream.read_records(sync_mode=SyncMode.full_refresh) next(records) @@ -49,7 +48,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: :param config: A Mapping of the user input configuration as defined in the connector spec. """ - authenticator = TokenAuthenticator(config[AUTH_KEY_FIELD]) + authenticator = TokenAuthenticator(config["secret_key"]) args = {"authenticator": authenticator, "start_date": config["start_date"]} incremental_args = {**args, "lookback_window_days": config.get("lookback_window_days")} return [ diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py b/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py index 41d2df2991e5..ed69c88c9249 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py @@ -9,11 +9,10 @@ import requests import pendulum from airbyte_cdk.sources.streams.http import HttpStream -from source_paystack.constants import PAYSTACK_API_BASE_URL, PAYSTACK_CREATED_AT, PAYSTACK_CREATED_AT_MISC class PaystackStream(HttpStream, ABC): - url_base = PAYSTACK_API_BASE_URL + url_base = "https://api.paystack.co/" primary_key = "id" def __init__(self, start_date: str, **kwargs): @@ -22,7 +21,8 @@ def __init__(self, start_date: str, **kwargs): def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: decoded_response = response.json() - page, pageCount = decoded_response['meta']['page'], response.json()['meta']['pageCount'] + page = decoded_response['meta']['page'] + pageCount = decoded_response['meta']['pageCount'] if page < pageCount: return { "page": page + 1 } @@ -106,7 +106,7 @@ class Customers(IncrementalPaystackStream): API docs: https://paystack.com/docs/api/#customer-list """ - cursor_field = PAYSTACK_CREATED_AT + cursor_field = "createdAt" def path(self, **kwargs) -> str: return "customer" @@ -117,7 +117,7 @@ class Disputes(IncrementalPaystackStream): API docs: https://paystack.com/docs/api/#dispute-list """ - cursor_field = PAYSTACK_CREATED_AT + cursor_field = "createdAt" def path(self, **kwargs) -> str: return "dispute" @@ -128,7 +128,7 @@ class Invoices(IncrementalPaystackStream): API docs: https://paystack.com/docs/api/#invoice-list """ - cursor_field = PAYSTACK_CREATED_AT_MISC + cursor_field = "created_at" def path(self, **kwargs) -> str: return "paymentrequest" @@ -139,7 +139,7 @@ class Refunds(IncrementalPaystackStream): API docs: https://paystack.com/docs/api/#refund-list """ - cursor_field = PAYSTACK_CREATED_AT + cursor_field = "createdAt" def path(self, **kwargs) -> str: return "refund" @@ -150,7 +150,7 @@ class Settlements(IncrementalPaystackStream): API docs: https://paystack.com/docs/api/#settlement """ - cursor_field = PAYSTACK_CREATED_AT + cursor_field = "createdAt" def path(self, **kwargs) -> str: return "settlement" @@ -161,7 +161,7 @@ class Subscriptions(IncrementalPaystackStream): API docs: https://paystack.com/docs/api/#subscription-list """ - cursor_field = PAYSTACK_CREATED_AT + cursor_field = "createdAt" def path(self, **kwargs) -> str: return "subscription" @@ -172,7 +172,7 @@ class Transactions(IncrementalPaystackStream): API docs: https://paystack.com/docs/api/#transaction-list """ - cursor_field = PAYSTACK_CREATED_AT + cursor_field = "createdAt" def path(self, **kwargs) -> str: return "transaction" @@ -183,7 +183,7 @@ class Transfers(IncrementalPaystackStream): API docs: https://paystack.com/docs/api/#transfer-list """ - cursor_field = PAYSTACK_CREATED_AT + cursor_field = "createdAt" def path(self, **kwargs) -> str: return "transfer" From aff3d4880dda75dd6e703a00ccfb8e0023013e3b Mon Sep 17 00:00:00 2001 From: Foluso Ogunlana Date: Sat, 23 Oct 2021 13:45:45 +0100 Subject: [PATCH 21/22] fix(67): correct "null, null" values in schemas --- .../sample_files/configured_catalog.json | 13 +++++++------ .../source_paystack/schemas/invoices.json | 5 +++-- .../source_paystack/schemas/subscriptions.json | 2 +- .../source_paystack/schemas/transactions.json | 4 ++-- .../source_paystack/schemas/transfers.json | 2 +- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/source-paystack/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-paystack/sample_files/configured_catalog.json index 1cec82dbc835..577660543d01 100644 --- a/airbyte-integrations/connectors/source-paystack/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-paystack/sample_files/configured_catalog.json @@ -205,7 +205,7 @@ "type": ["null", "string"] }, "pdf_url": { - "type": ["null", "null"] + "type": ["null", "string"] }, "line_items": { "type": ["null", "array"], @@ -252,7 +252,8 @@ "format": "date-time" }, "metadata": { - "type": ["null", "null"] + "type": ["null", "object"], + "properties": {} }, "notifications": { "type": ["null", "array"], @@ -476,7 +477,7 @@ "type": ["null", "integer"] }, "split_code": { - "type": ["null", "null"] + "type": ["null", "string"] }, "payments_count": { "type": ["null", "integer"] @@ -593,7 +594,7 @@ "type": ["null", "integer"] }, "fees_split": { - "type": ["null", "null"] + "type": ["null", "string"] }, "customer": { "type": ["null", "object"], @@ -639,7 +640,7 @@ "type": ["null", "string"] }, "identifier": { - "type": ["null", "null"] + "type": ["null", "string"] }, "entry_point": { "type": ["null", "string"] @@ -700,7 +701,7 @@ "type": ["null", "string"] }, "account_name": { - "type": ["null", "null"] + "type": ["null", "string"] }, "bank_code": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/invoices.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/invoices.json index d8ef0d8647ef..ae0f78448a2b 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/invoices.json +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/invoices.json @@ -30,7 +30,7 @@ "type": ["null", "string"] }, "pdf_url": { - "type": ["null", "null"] + "type": ["null", "string"] }, "line_items": { "type": ["null", "array"], @@ -77,7 +77,8 @@ "format": "date-time" }, "metadata": { - "type": ["null", "null"] + "type": ["null", "object"], + "properties": {} }, "notifications": { "type": ["null", "array"], diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/subscriptions.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/subscriptions.json index 190a9e3647f9..5fd5626d3b52 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/subscriptions.json +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/subscriptions.json @@ -58,7 +58,7 @@ "type": ["null", "integer"] }, "split_code": { - "type": ["null", "null"] + "type": ["null", "string"] }, "payments_count": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transactions.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transactions.json index 8372a2df86d4..8df72eae4ed9 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transactions.json +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transactions.json @@ -95,7 +95,7 @@ "type": ["null", "integer"] }, "fees_split": { - "type": ["null", "null"] + "type": ["null", "string"] }, "customer": { "type": ["null", "object"], @@ -141,7 +141,7 @@ "type": ["null", "string"] }, "identifier": { - "type": ["null", "null"] + "type": ["null", "string"] }, "entry_point": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transfers.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transfers.json index fe5658d4abfc..38f116ec2a8a 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transfers.json +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/transfers.json @@ -41,7 +41,7 @@ "type": ["null", "string"] }, "account_name": { - "type": ["null", "null"] + "type": ["null", "string"] }, "bank_code": { "type": ["null", "string"] From 62c3fe542860efada612c31c5cbf16ea4f2f0c7f Mon Sep 17 00:00:00 2001 From: Foluso Ogunlana Date: Sat, 23 Oct 2021 15:11:43 +0100 Subject: [PATCH 22/22] chore(67): update file formatting with gradle format --- .../source-paystack/sample_files/state.json | 16 ++++---- .../connectors/source-paystack/setup.py | 7 +--- .../source_paystack/schemas/customers.json | 2 +- .../source_paystack/schemas/invoices.json | 14 +++---- .../source-paystack/source_paystack/source.py | 17 ++------- .../source_paystack/streams.py | 28 +++++--------- .../unit_tests/test_incremental_streams.py | 38 ++++++++----------- .../source-paystack/unit_tests/test_source.py | 29 +++++++------- .../unit_tests/test_streams.py | 9 ++++- 9 files changed, 69 insertions(+), 91 deletions(-) diff --git a/airbyte-integrations/connectors/source-paystack/sample_files/state.json b/airbyte-integrations/connectors/source-paystack/sample_files/state.json index 50fa6cde6361..2b7a6496036c 100644 --- a/airbyte-integrations/connectors/source-paystack/sample_files/state.json +++ b/airbyte-integrations/connectors/source-paystack/sample_files/state.json @@ -1,10 +1,10 @@ { - "customers": { "createdAt": "2019-08-01T00:00:00Z" }, - "disputes": { "createdAt": "2019-08-01T00:00:00Z" }, - "invoices": { "created_at": "2019-08-01T00:00:00Z" }, - "refunds": { "createdAt": "2019-08-01T00:00:00Z" }, - "settlements": { "createdAt": "2019-08-01T00:00:00Z" }, - "subscriptions": { "createdAt": "2019-08-01T00:00:00Z" }, - "transactions": { "createdAt": "2019-08-01T00:00:00Z" }, - "transfers": { "createdAt": "2019-08-01T00:00:00Z" } + "customers": { "createdAt": "2019-08-01T00:00:00Z" }, + "disputes": { "createdAt": "2019-08-01T00:00:00Z" }, + "invoices": { "created_at": "2019-08-01T00:00:00Z" }, + "refunds": { "createdAt": "2019-08-01T00:00:00Z" }, + "settlements": { "createdAt": "2019-08-01T00:00:00Z" }, + "subscriptions": { "createdAt": "2019-08-01T00:00:00Z" }, + "transactions": { "createdAt": "2019-08-01T00:00:00Z" }, + "transfers": { "createdAt": "2019-08-01T00:00:00Z" } } diff --git a/airbyte-integrations/connectors/source-paystack/setup.py b/airbyte-integrations/connectors/source-paystack/setup.py index 9f32194ae49f..cd0cc49d0534 100644 --- a/airbyte-integrations/connectors/source-paystack/setup.py +++ b/airbyte-integrations/connectors/source-paystack/setup.py @@ -9,12 +9,7 @@ "airbyte-cdk", ] -TEST_REQUIREMENTS = [ - "pytest~=6.1", - "pytest-mock~=3.6.1", - "source-acceptance-test", - "requests-mock" -] +TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6.1", "source-acceptance-test", "requests-mock"] setup( name="source_paystack", diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json index 7cc9020ad0a4..e6566415311b 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/customers.json @@ -40,5 +40,5 @@ "type": ["null", "string"], "format": "date-time" } - } + } } diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/invoices.json b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/invoices.json index ae0f78448a2b..c48220646a68 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/invoices.json +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/schemas/invoices.json @@ -85,14 +85,14 @@ "items": { "type": ["null", "object"], "properties": { - "sent_at": { - "type": ["null", "string"] - }, - "channel": { - "type": ["null", "string"] - } + "sent_at": { + "type": ["null", "string"] + }, + "channel": { + "type": ["null", "string"] + } } - } + } }, "offline_reference": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/source.py b/airbyte-integrations/connectors/source-paystack/source_paystack/source.py index a2127ec2ccd2..2a4c5517106d 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/source.py +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/source.py @@ -1,23 +1,15 @@ # # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # + from typing import Any, List, Mapping, Tuple -import pendulum from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -from source_paystack.streams import ( - Customers, - Disputes, - Invoices, - Refunds, - Settlements, - Subscriptions, - Transactions, - Transfers -) +from source_paystack.streams import Customers, Disputes, Invoices, Refunds, Settlements, Subscriptions, Transactions, Transfers + class SourcePaystack(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: @@ -59,6 +51,5 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Settlements(**incremental_args), Subscriptions(**incremental_args), Transactions(**incremental_args), - Transfers(**incremental_args) + Transfers(**incremental_args), ] - diff --git a/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py b/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py index ed69c88c9249..3053f1865184 100644 --- a/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py +++ b/airbyte-integrations/connectors/source-paystack/source_paystack/streams.py @@ -1,13 +1,13 @@ # # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # + import math from abc import ABC, abstractmethod from typing import Any, Iterable, Mapping, MutableMapping, Optional -from datetime import datetime -import requests import pendulum +import requests from airbyte_cdk.sources.streams.http import HttpStream @@ -21,17 +21,14 @@ def __init__(self, start_date: str, **kwargs): def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: decoded_response = response.json() - page = decoded_response['meta']['page'] - pageCount = decoded_response['meta']['pageCount'] + page = decoded_response["meta"]["page"] + pageCount = decoded_response["meta"]["pageCount"] if page < pageCount: - return { "page": page + 1 } + return {"page": page + 1} def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, any] = None, - next_page_token: Mapping[str, Any] = None + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: params = {"perPage": 200} @@ -61,11 +58,7 @@ def cursor_field(self) -> str: """ pass - def get_updated_state( - self, - current_stream_state: MutableMapping[str, Any], - latest_record: Mapping[str, Any] - ) -> Mapping[str, Any]: + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: """ Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object and returning an updated state object. @@ -75,7 +68,7 @@ def get_updated_state( self.cursor_field: max( latest_record_created, current_stream_state.get(self.cursor_field, None), - key=lambda d: pendulum.parse(d).int_timestamp if d else 0 + key=lambda d: pendulum.parse(d).int_timestamp if d else 0, ) } @@ -89,10 +82,7 @@ def _get_start_date(self, stream_state) -> str: start_point = self.start_date if stream_state and self.cursor_field in stream_state: stream_record_created = stream_state[self.cursor_field] - start_point = max( - start_point, - pendulum.parse(stream_record_created).int_timestamp - ) + start_point = max(start_point, pendulum.parse(stream_record_created).int_timestamp) if start_point and self.lookback_window_days: self.logger.info(f"Applying lookback window of {self.lookback_window_days} days to stream {self.name}") diff --git a/airbyte-integrations/connectors/source-paystack/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-paystack/unit_tests/test_incremental_streams.py index db2af442f3aa..625e286fc0a5 100644 --- a/airbyte-integrations/connectors/source-paystack/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-paystack/unit_tests/test_incremental_streams.py @@ -2,17 +2,16 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # -import uuid import math +import uuid import pytest -import pendulum -from airbyte_cdk.models import SyncMode from pytest import fixture -from source_paystack.streams import IncrementalPaystackStream, Customers +from source_paystack.streams import IncrementalPaystackStream START_DATE = "2020-08-01T00:00:00Z" + @fixture def patch_incremental_base_class(mocker): # Mock abstract methods to enable instantiating abstract class @@ -24,38 +23,32 @@ def patch_incremental_base_class(mocker): def test_get_updated_state_uses_timestamp_of_latest_record(patch_incremental_base_class): stream = IncrementalPaystackStream(start_date=START_DATE) - inputs = { - "current_stream_state": {stream.cursor_field: "2021-08-01"}, - "latest_record": {stream.cursor_field: "2021-08-02T00:00:00Z"} - } + inputs = {"current_stream_state": {stream.cursor_field: "2021-08-01"}, "latest_record": {stream.cursor_field: "2021-08-02T00:00:00Z"}} updated_state = stream.get_updated_state(**inputs) assert updated_state == {stream.cursor_field: "2021-08-02T00:00:00Z"} + def test_get_updated_state_returns_current_state_for_old_records(patch_incremental_base_class): stream = IncrementalPaystackStream(start_date=START_DATE) current_state = {stream.cursor_field: "2021-08-03"} - inputs = { - "current_stream_state": current_state, - "latest_record": {stream.cursor_field: "2021-08-02T00:00:00Z"} - } + inputs = {"current_stream_state": current_state, "latest_record": {stream.cursor_field: "2021-08-02T00:00:00Z"}} updated_state = stream.get_updated_state(**inputs) assert updated_state == current_state + def test_get_updated_state_uses_timestamp_of_latest_record_when_no_current_state_exists(patch_incremental_base_class): stream = IncrementalPaystackStream(start_date=START_DATE) - inputs = { - "current_stream_state": {}, - "latest_record": {stream.cursor_field: "2021-08-02T00:00:00Z"} - } + inputs = {"current_stream_state": {}, "latest_record": {stream.cursor_field: "2021-08-02T00:00:00Z"}} updated_state = stream.get_updated_state(**inputs) assert updated_state == {stream.cursor_field: "2021-08-02T00:00:00Z"} + def test_request_params_includes_incremental_start_point(patch_incremental_base_class): stream = IncrementalPaystackStream(start_date=START_DATE) inputs = { @@ -68,6 +61,7 @@ def test_request_params_includes_incremental_start_point(patch_incremental_base_ assert params == {"perPage": 200, "page": 37, "from": "2021-09-02T00:00:00Z"} + @pytest.mark.parametrize( "lookback_window_days, current_state, expected, message", [ @@ -78,11 +72,7 @@ def test_request_params_includes_incremental_start_point(patch_incremental_base_ ], ) def test_request_params_incremental_start_point_applies_lookback_window( - patch_incremental_base_class, - lookback_window_days, - current_state, - expected, - message + patch_incremental_base_class, lookback_window_days, current_state, expected, message ): stream = IncrementalPaystackStream(start_date=START_DATE, lookback_window_days=lookback_window_days) inputs = { @@ -96,6 +86,7 @@ def test_request_params_incremental_start_point_applies_lookback_window( assert params["perPage"] == 200 assert params["from"] == expected, message + def test_request_params_incremental_start_point_defaults_to_start_date(patch_incremental_base_class): stream = IncrementalPaystackStream(start_date=START_DATE) inputs = {"stream_slice": None, "next_page_token": None, "stream_state": None} @@ -104,14 +95,17 @@ def test_request_params_incremental_start_point_defaults_to_start_date(patch_inc assert params == {"perPage": 200, "from": "2020-08-01T00:00:00Z"} + def test_supports_incremental(patch_incremental_base_class): stream = IncrementalPaystackStream(start_date=START_DATE) assert stream.supports_incremental + def test_source_defined_cursor(patch_incremental_base_class): stream = IncrementalPaystackStream(start_date=START_DATE) assert stream.source_defined_cursor + def test_stream_checkpoint_interval(patch_incremental_base_class): stream = IncrementalPaystackStream(start_date=START_DATE) - assert stream.state_checkpoint_interval == math.inf \ No newline at end of file + assert stream.state_checkpoint_interval == math.inf diff --git a/airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py b/airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py index f4936918d1d1..1f7c000c3623 100644 --- a/airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-paystack/unit_tests/test_source.py @@ -2,40 +2,40 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # -from unittest.mock import MagicMock, ANY +from unittest.mock import ANY, MagicMock import pytest import requests from source_paystack.source import SourcePaystack + @pytest.mark.parametrize( ("items",), - [ - ([{ "createdAt": "2022-07-01T00:00:00Z", "id": 12345 }],), # single customer - ([],) # no customers - ], + [([{"createdAt": "2022-07-01T00:00:00Z", "id": 12345}],), ([],)], # single customer #  no customers ) def test_check_connection_success(mocker, requests_mock, items): source = SourcePaystack() logger_mock = MagicMock() - config_mock = { - "start_date": "2020-07-01T00:00:00Z", - "secret_key": "sk_test_abc123" - } - requests_mock.get("https://api.paystack.co/customer", json={ - "data": items, - "meta": {"page": 1, "pageCount": 0}, - }) + config_mock = {"start_date": "2020-07-01T00:00:00Z", "secret_key": "sk_test_abc123"} + requests_mock.get( + "https://api.paystack.co/customer", + json={ + "data": items, + "meta": {"page": 1, "pageCount": 0}, + }, + ) assert source.check_connection(logger_mock, config_mock) == (True, None) + def test_check_connection_failure(mocker, requests_mock): source = SourcePaystack() logger_mock, config_mock = MagicMock(), MagicMock() - requests_mock.get("https://api.paystack.co/customer", json={ "status": False, "message": "Failed" }) + requests_mock.get("https://api.paystack.co/customer", json={"status": False, "message": "Failed"}) assert source.check_connection(logger_mock, config_mock) == (False, ANY) + def test_check_connection_error(mocker, requests_mock): source = SourcePaystack() logger_mock, config_mock = MagicMock(), MagicMock() @@ -43,6 +43,7 @@ def test_check_connection_error(mocker, requests_mock): assert source.check_connection(logger_mock, config_mock) == (False, ANY) + def test_streams(mocker): source = SourcePaystack() streams = source.streams({"start_date": "2020-08-01", "secret_key": "sk_test_123456"}) diff --git a/airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py index b999d43e4fcc..60bf435a49ba 100644 --- a/airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-paystack/unit_tests/test_streams.py @@ -10,6 +10,7 @@ START_DATE = "2020-08-01T00:00:00Z" + @pytest.fixture def patch_base_class(mocker): # Mock abstract methods to enable instantiating abstract class @@ -26,6 +27,7 @@ def test_request_params_includes_pagination_limit(patch_base_class): assert params == {"perPage": 200} + def test_request_params_for_includes_page_number_for_pagination(patch_base_class): stream = PaystackStream(start_date=START_DATE) inputs = {"stream_slice": None, "stream_state": None, "next_page_token": {"page": 2}} @@ -34,6 +36,7 @@ def test_request_params_for_includes_page_number_for_pagination(patch_base_class assert params == {"perPage": 200, "page": 2} + def test_next_page_token_increments_page_number(patch_base_class): stream = PaystackStream(start_date=START_DATE) mock_response = MagicMock() @@ -44,6 +47,7 @@ def test_next_page_token_increments_page_number(patch_base_class): assert token == {"page": 3} + def test_next_page_token_is_none_when_last_page_reached(patch_base_class): stream = PaystackStream(start_date=START_DATE) mock_response = MagicMock() @@ -54,6 +58,7 @@ def test_next_page_token_is_none_when_last_page_reached(patch_base_class): assert token is None + def test_next_page_token_is_none_when_no_pages_exist(patch_base_class): stream = PaystackStream(start_date=START_DATE) mock_response = MagicMock() @@ -64,18 +69,20 @@ def test_next_page_token_is_none_when_no_pages_exist(patch_base_class): assert token is None + def test_parse_response_generates_data(patch_base_class): stream = PaystackStream(start_date=START_DATE) mock_response = MagicMock() mock_response.json.return_value = {"data": [{"id": 1137850082}, {"id": 1137850097}]} inputs = {"response": mock_response} - + parsed = stream.parse_response(**inputs) first, second = next(parsed), next(parsed) assert first == {"id": 1137850082} assert second == {"id": 1137850097} + @pytest.mark.parametrize( ("http_status", "should_retry"), [