From f390c898e41b708f1c6e492240f0193a981778e0 Mon Sep 17 00:00:00 2001 From: Serhii Date: Tue, 2 Aug 2022 21:36:10 +0300 Subject: [PATCH 1/3] Increased unit test coverage --- .../connectors/source-harvest/README.md | 2 +- .../source-harvest/requirements.txt | 3 + .../connectors/source-harvest/setup.py | 2 + .../source-harvest/unit_tests/conftest.py | 41 ++++++++++ .../source-harvest/unit_tests/unit_test.py | 79 +++++++++++++++++- docs/integrations/sources/harvest.md | 80 +++++++++++-------- .../sources/paypal-transaction.md | 8 +- 7 files changed, 175 insertions(+), 40 deletions(-) create mode 100644 airbyte-integrations/connectors/source-harvest/requirements.txt create mode 100644 airbyte-integrations/connectors/source-harvest/unit_tests/conftest.py diff --git a/airbyte-integrations/connectors/source-harvest/README.md b/airbyte-integrations/connectors/source-harvest/README.md index ea44019174c2..0d0b500167f4 100644 --- a/airbyte-integrations/connectors/source-harvest/README.md +++ b/airbyte-integrations/connectors/source-harvest/README.md @@ -79,7 +79,7 @@ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integrat Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. First install test dependencies into your virtual environment: ``` -pip install .[tests] +pip install .'[tests]' ``` ### Unit Tests To run unit tests locally, from the connector directory run: diff --git a/airbyte-integrations/connectors/source-harvest/requirements.txt b/airbyte-integrations/connectors/source-harvest/requirements.txt new file mode 100644 index 000000000000..7be17a56d745 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/requirements.txt @@ -0,0 +1,3 @@ +# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-harvest/setup.py b/airbyte-integrations/connectors/source-harvest/setup.py index f7bc9e62a88c..25cfab38fd2f 100644 --- a/airbyte-integrations/connectors/source-harvest/setup.py +++ b/airbyte-integrations/connectors/source-harvest/setup.py @@ -11,6 +11,8 @@ TEST_REQUIREMENTS = [ "pytest~=6.1", + "requests-mock", + "source-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-harvest/unit_tests/conftest.py b/airbyte-integrations/connectors/source-harvest/unit_tests/conftest.py new file mode 100644 index 000000000000..0c393ce9ea0b --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/unit_tests/conftest.py @@ -0,0 +1,41 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from pendulum import parse +from pytest import fixture + + +@fixture(name="config") +def config_fixture(requests_mock): + url = "https://id.getharvest.com/api/v2/oauth2/token" + requests_mock.get(url, json={}) + + config = { + "account_id": "ID", + "replication_start_date": "2021-01-01T21:20:07Z", + "credentials": { + "api_token": "TOKEN" + } + } + + return config + + +@fixture(name="replication_start_date") +def replication_start_date_fixture(config): + return parse(config["replication_start_date"]) + + +@fixture(name="from_date") +def from_date_fixture(replication_start_date): + return replication_start_date.date() + + +@fixture(name="mock_stream") +def mock_stream_fixture(requests_mock): + def _mock_stream(path, response={}): + url = f"https://api.harvestapp.com/v2/{path}" + requests_mock.get(url, json=response) + return _mock_stream + diff --git a/airbyte-integrations/connectors/source-harvest/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-harvest/unit_tests/unit_test.py index dddaea0060fa..68545d3b09e7 100644 --- a/airbyte-integrations/connectors/source-harvest/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-harvest/unit_tests/unit_test.py @@ -2,6 +2,81 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +import requests +from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.sources.streams.http.auth import NoAuth +from source_harvest.source import SourceHarvest +from source_harvest.streams import HarvestStream, InvoicePayments, ExpensesClients -def test_example_method(): - assert True +logger = AirbyteLogger() + + +def test_check_connection_ok(config, mock_stream): + mock_stream("users", response={"users": [{"id": 1}], "next_page": 2}) + ok, error_msg = SourceHarvest().check_connection(logger, config=config) + + assert ok + assert not error_msg + + +def test_check_connection_empty_config(config): + config = {} + + ok, error_msg = SourceHarvest().check_connection(logger, config=config) + + assert not ok + assert error_msg + + +def test_check_connection_invalid_config(config): + config.pop("replication_start_date") + ok, error_msg = SourceHarvest().check_connection(logger, config=config) + + assert not ok + assert error_msg + + +def test_check_connection_exception(config): + ok, error_msg = SourceHarvest().check_connection(logger, config=config) + + assert not ok + assert error_msg + + +def test_streams(config): + streams = SourceHarvest().streams(config) + + assert len(streams) == 32 + + +def test_next_page_token(config, mocker): + next_page = 2 + expected = {"page": next_page} + + instance = HarvestStream(authenticator=NoAuth()) + + response = mocker.Mock(spec=requests.Response, request=mocker.Mock(spec=requests.Request)) + response.json.return_value = {"next_page": next_page} + + assert instance.next_page_token(response) == expected + + +def test_child_stream_slices(config, replication_start_date, mock_stream): + object_id = 1 + mock_stream("invoices", response={"invoices": [{"id": object_id}]}) + mock_stream(f"invoices/{object_id}/payments", {"invoice_payments": [{"id": object_id}]}) + + invoice_payments_instance = InvoicePayments(authenticator=NoAuth(), replication_start_date=replication_start_date) + stream_slice = next(invoice_payments_instance.stream_slices(sync_mode=None)) + invoice_payments = invoice_payments_instance.read_records(sync_mode=None, stream_slice=stream_slice) + + assert next(invoice_payments) + + +def test_report_base_stream(config, from_date, mock_stream): + mock_stream("reports/expenses/clients", response={"results": [{"client_id": 1}]}) + + invoice_payments_instance = ExpensesClients(authenticator=NoAuth(), from_date=from_date) + invoice_payments = invoice_payments_instance.read_records(sync_mode=None) + + assert next(invoice_payments) diff --git a/docs/integrations/sources/harvest.md b/docs/integrations/sources/harvest.md index 6349fcaf35fd..895e046804a8 100644 --- a/docs/integrations/sources/harvest.md +++ b/docs/integrations/sources/harvest.md @@ -1,12 +1,54 @@ # Harvest -## Overview +This page contains the setup guide and reference information for the Harvest source connector. -The Harvest connector can be used to sync your Harvest data. It supports full refresh sync for all streams and incremental sync for all streams except of Expense Reports streams which are: Clients Report, Projects Report, Categories Report, Team Report. Incremental sync is also now available for Company stream, but it always has only one record. +## Prerequisites + +See [docs](https://help.getharvest.com/api-v2/authentication-api/authentication/authentication/) for more details. + +## Setup guide +### Step 1: Set up Harvest + +This connector supports only authentication with API Key. To obtain API key follow the instructions below: + +1. Go to Account Settings page; +2. Under Integrations section press Authorized OAuth2 API Clients button; +3. New page will be opened on which you need to click on Create New Personal Access Token button and follow instructions. + +## Step 2: Set up the Harvest connector in Airbyte + +### For Airbyte Cloud: + +1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**. +3. On the Set up the source page, enter the name for the Harvest connector and select **Harvest** from the Source type dropdown. +4. For Airbyte Cloud, click **Authenticate your Harvest account** to sign in with Harvest and authorize your account. +5. Enter your `account_id` +6. Enter the `replication_start_date` you want your sync to start from +7. Click **Set up source** + +### For Airbyte OSS: +1. Navigate to the Airbyte Open Source dashboard +2. Set the name for your source +3. Enter your `api_token` +4. Enter your `account_id` +5. Enter the `replication_start_date` you want your sync to start from +6. Click **Set up source** + +## Supported sync modes + +The Harvest source connector supports the following[ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + +| Feature | Supported? | +| :--- | :--- | +| Full Refresh Sync | Yes | +| Incremental Sync | Yes | +| Replicate Incremental Deletes | No | +| SSL connection | Yes | +| Namespaces | No | -### Output schema -Several output streams are available from this source: +## Supported Streams * [Client Contacts](https://help.getharvest.com/api-v2/clients-api/clients/contacts/) \(Incremental\) * [Clients](https://help.getharvest.com/api-v2/clients-api/clients/clients/) \(Incremental\) @@ -34,37 +76,10 @@ Several output streams are available from this source: * [Time Reports](https://help.getharvest.com/api-v2/reports-api/reports/time-reports/) * [Project Budget Report](https://help.getharvest.com/api-v2/reports-api/reports/project-budget-report/) -### Features - -| Feature | Supported? | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental Sync | Yes | -| Replicate Incremental Deletes | No | -| SSL connection | Yes | -| Namespaces | No | - -### Performance considerations +## Performance considerations The Harvest connector will gracefully handle rate limits. For more information, see [the Harvest docs for rate limitations](https://help.getharvest.com/api-v2/introduction/overview/general/#rate-limiting). -## Getting started - -### Requirements - -* Harvest Account -* Harvest Authorized OAuth2 API Client to create Access Token and get account ID - -### Setup guide - -This connector supports only authentication with API Key. To obtain API key follow the instructions below: - -1. Go to Account Settings page; -2. Under Integrations section press Authorized OAuth2 API Clients button; -3. New page will be opened on which you need to click on Create New Personal Access Token button and follow instructions. - -See [docs](https://help.getharvest.com/api-v2/authentication-api/authentication/authentication/) for more details. - ## Changelog | Version | Date | Pull Request | Subject | @@ -77,4 +92,3 @@ See [docs](https://help.getharvest.com/api-v2/authentication-api/authentication/ | 0.1.2 | 2021-06-07 | [4222](https://github.com/airbytehq/airbyte/pull/4222) | Correct specification parameter name | | 0.1.1 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | | 0.1.0 | 2021-06-07 | [3709](https://github.com/airbytehq/airbyte/pull/3709) | Release Harvest connector! | - diff --git a/docs/integrations/sources/paypal-transaction.md b/docs/integrations/sources/paypal-transaction.md index fc333d29bc32..d52605b29f95 100644 --- a/docs/integrations/sources/paypal-transaction.md +++ b/docs/integrations/sources/paypal-transaction.md @@ -25,11 +25,11 @@ In order to get an `Client ID` and `Secret` please go to [this](https://develope ### For Airbyte OSS: 1. Navigate to the Airbyte Open Source dashboard 2. Set the name for your source -4. Enter your client id +3. Enter your client id 4. Enter your secret -4. Choose if your account is sandbox -5. Enter the date you want your sync to start from -6. Click **Set up source** +5. Choose if your account is sandbox +6. Enter the date you want your sync to start from +7. Click **Set up source** ## Supported sync modes From 8faa8058b5d55d67f352af8f7c95a44e6ab75abe Mon Sep 17 00:00:00 2001 From: Serhii Date: Mon, 8 Aug 2022 14:49:36 +0300 Subject: [PATCH 2/3] Increased unit test --- .../connectors/source-harvest/unit_tests/unit_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-harvest/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-harvest/unit_tests/unit_test.py index 582b3fba1785..5138f68da87f 100644 --- a/airbyte-integrations/connectors/source-harvest/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-harvest/unit_tests/unit_test.py @@ -77,6 +77,7 @@ def test_report_base_stream(config, from_date, mock_stream): mock_stream("reports/expenses/clients", response={"results": [{"client_id": 1}]}) invoice_payments_instance = ExpensesClients(authenticator=NoAuth(), from_date=from_date) - invoice_payments = invoice_payments_instance.read_records(sync_mode=None) + stream_slice = next(invoice_payments_instance.stream_slices(sync_mode=None)) + invoice_payments = invoice_payments_instance.read_records(sync_mode=None, stream_slice=stream_slice) assert next(invoice_payments) From ef9f0a9c2e1a7c12ff396bd53ed57cb7ba288de6 Mon Sep 17 00:00:00 2001 From: Serhii Date: Mon, 8 Aug 2022 14:56:38 +0300 Subject: [PATCH 3/3] Updated to linter --- .../integration_tests/configured_catalog_report.json | 7 ++++++- .../connectors/source-harvest/unit_tests/conftest.py | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_report.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_report.json index 26d697a48e1d..ded3a860befb 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_report.json +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_report.json @@ -5,7 +5,12 @@ "name": "sponsored_products_report_stream", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["profileId"], ["recordType"], ["reportDate"], ["updatedAt"]] + "source_defined_primary_key": [ + ["profileId"], + ["recordType"], + ["reportDate"], + ["updatedAt"] + ] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite" diff --git a/airbyte-integrations/connectors/source-harvest/unit_tests/conftest.py b/airbyte-integrations/connectors/source-harvest/unit_tests/conftest.py index f3a488a8f0c4..b0962911b040 100644 --- a/airbyte-integrations/connectors/source-harvest/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-harvest/unit_tests/conftest.py @@ -30,4 +30,5 @@ def mock_stream_fixture(requests_mock): def _mock_stream(path, response={}): url = f"https://api.harvestapp.com/v2/{path}" requests_mock.get(url, json=response) + return _mock_stream