From 024b89ee91168bba9adaeaa092e514f3211a0e8b Mon Sep 17 00:00:00 2001 From: auganbay Date: Fri, 4 Feb 2022 15:47:53 +0600 Subject: [PATCH 1/9] check if stream exists in source --- .../source-hubspot/source_hubspot/source.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index deed1d336c23..7b925ea0fb69 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -2,11 +2,23 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # +import logging +from typing import Any, MutableMapping -from airbyte_cdk.sources.deprecated.base_source import BaseSource +from airbyte_cdk.sources.deprecated.base_source import BaseSource, BaseClient +from airbyte_cdk.models import ConfiguredAirbyteStream from .client import Client class SourceHubspot(BaseSource): client_class = Client + + def _read_stream( + self, logger: logging.Logger, client: BaseClient, configured_stream: ConfiguredAirbyteStream, state: MutableMapping[str, Any] + ): + stream_name = configured_stream.stream.name + if not client._apis.get(stream_name): + logger.warning(f"Stream {stream_name} is not in the source.") + return + yield from super()._read_stream(logger=logger, client=client, configured_stream=configured_stream, state=state) From 6c7bcba438223531c4bf533bd92dacf9dd2a63e2 Mon Sep 17 00:00:00 2001 From: auganbay Date: Fri, 4 Feb 2022 15:55:25 +0600 Subject: [PATCH 2/9] check if stream exists in source, added comment --- .../connectors/source-hubspot/source_hubspot/source.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index 7b925ea0fb69..3a7c73ef3735 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -17,8 +17,11 @@ class SourceHubspot(BaseSource): def _read_stream( self, logger: logging.Logger, client: BaseClient, configured_stream: ConfiguredAirbyteStream, state: MutableMapping[str, Any] ): + """ + This method is overridden to check if the stream exists in the client. + """ stream_name = configured_stream.stream.name if not client._apis.get(stream_name): - logger.warning(f"Stream {stream_name} is not in the source.") + logger.warning(f"Stream {stream_name} does not exist in the client.") return yield from super()._read_stream(logger=logger, client=client, configured_stream=configured_stream, state=state) From 5b9c5ecb95688ee3b08cf828ffda25adc72c52f3 Mon Sep 17 00:00:00 2001 From: auganbay Date: Fri, 4 Feb 2022 17:57:40 +0600 Subject: [PATCH 3/9] test skipping reading quotes stream --- .../source-hubspot/unit_tests/test_client.py | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py index d40b0f292d14..c8caefaa826a 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py @@ -6,11 +6,17 @@ from functools import partial import pytest +import logging + +from airbyte_cdk.models import Type, ConfiguredAirbyteCatalog from source_hubspot.api import API, PROPERTIES_PARAM_MAX_LENGTH, split_properties from source_hubspot.client import Client +from source_hubspot.source import SourceHubspot NUMBER_OF_PROPERTIES = 2000 +logger = logging.getLogger('test_client') + @pytest.fixture(name="some_credentials") def some_credentials_fixture(): @@ -260,3 +266,51 @@ def test_stream_with_splitting_properties_with_new_record(self, requests_mock, c stream_records = list(test_stream.read(getter=partial(self.get, test_stream.url, api=api))) assert len(stream_records) == 6 + + +@pytest.fixture(name="oauth_config") +def oauth_config_fixture(): + return { + "start_date": "2021-10-10T00:00:00Z", + "credentials": { + "credentials_title": "OAuth Credentials", + "redirect_uri": "https://airbyte.io", + "client_id": "test_client_id", + "client_secret": "test_client_secret", + "refresh_token": "test_refresh_token", + "access_token": "test_access_token", + "token_expires": "2021-05-30T06:00:00Z" + } + } + + +@pytest.fixture(name='configured_catalog') +def configured_catalog_fixture(): + configured_catalog = { + "streams": [ + { + "stream": { + "name": "quotes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": True, + "default_cursor_field": ["updatedAt"] + }, + "sync_mode": "incremental", + "cursor_field": ["updatedAt"], + "destination_sync_mode": "append" + } + ] + } + return ConfiguredAirbyteCatalog.parse_obj(configured_catalog) + + +def test_it_should_not_read_quotes_stream_if_it_does_not_exist_in_client(oauth_config, configured_catalog): + """ + If 'quotes' stream is not in the client, it should skip it. + """ + source = SourceHubspot() + + all_records = list(source.read(logger, config=oauth_config, catalog=configured_catalog, state=None)) + records = [record for record in all_records if record.type == Type.RECORD] + assert not records From add55e37d76fc1ebf929d9f56114702358839983 Mon Sep 17 00:00:00 2001 From: auganbay Date: Fri, 4 Feb 2022 18:08:15 +0600 Subject: [PATCH 4/9] format code --- .../source-hubspot/source_hubspot/source.py | 2 +- .../source-hubspot/unit_tests/test_client.py | 41 +++++++++---------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index 3a7c73ef3735..b6c74eaea9b9 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -5,8 +5,8 @@ import logging from typing import Any, MutableMapping -from airbyte_cdk.sources.deprecated.base_source import BaseSource, BaseClient from airbyte_cdk.models import ConfiguredAirbyteStream +from airbyte_cdk.sources.deprecated.base_source import BaseClient, BaseSource from .client import Client diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py index c8caefaa826a..cc738ae78862 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py @@ -3,19 +3,18 @@ # +import logging from functools import partial import pytest -import logging - -from airbyte_cdk.models import Type, ConfiguredAirbyteCatalog +from airbyte_cdk.models import ConfiguredAirbyteCatalog, Type from source_hubspot.api import API, PROPERTIES_PARAM_MAX_LENGTH, split_properties from source_hubspot.client import Client from source_hubspot.source import SourceHubspot NUMBER_OF_PROPERTIES = 2000 -logger = logging.getLogger('test_client') +logger = logging.getLogger("test_client") @pytest.fixture(name="some_credentials") @@ -279,28 +278,28 @@ def oauth_config_fixture(): "client_secret": "test_client_secret", "refresh_token": "test_refresh_token", "access_token": "test_access_token", - "token_expires": "2021-05-30T06:00:00Z" - } + "token_expires": "2021-05-30T06:00:00Z", + }, } -@pytest.fixture(name='configured_catalog') +@pytest.fixture(name="configured_catalog") def configured_catalog_fixture(): configured_catalog = { - "streams": [ - { - "stream": { - "name": "quotes", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": True, - "default_cursor_field": ["updatedAt"] - }, - "sync_mode": "incremental", - "cursor_field": ["updatedAt"], - "destination_sync_mode": "append" - } - ] + "streams": [ + { + "stream": { + "name": "quotes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": True, + "default_cursor_field": ["updatedAt"], + }, + "sync_mode": "incremental", + "cursor_field": ["updatedAt"], + "destination_sync_mode": "append", + } + ] } return ConfiguredAirbyteCatalog.parse_obj(configured_catalog) From 381550c7ef6b4d244c0411b6ac3520547e2d4387 Mon Sep 17 00:00:00 2001 From: auganbay Date: Mon, 7 Feb 2022 12:40:17 +0600 Subject: [PATCH 5/9] airbyte-cdk version --- airbyte-integrations/connectors/source-hubspot/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-hubspot/setup.py b/airbyte-integrations/connectors/source-hubspot/setup.py index 2d4955656504..46fd1ff67bfe 100644 --- a/airbyte-integrations/connectors/source-hubspot/setup.py +++ b/airbyte-integrations/connectors/source-hubspot/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", "backoff==1.11.1", "pendulum==2.1.2", "requests==2.26.0", From 07ff9acdaae08813c4985bfbd9d85a47b70490f2 Mon Sep 17 00:00:00 2001 From: auganbay Date: Mon, 7 Feb 2022 14:13:07 +0600 Subject: [PATCH 6/9] added __init__.py to unit_tests --- airbyte-integrations/connectors/source-hubspot/setup.py | 2 +- .../connectors/source-hubspot/unit_tests/__init__.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/__init__.py diff --git a/airbyte-integrations/connectors/source-hubspot/setup.py b/airbyte-integrations/connectors/source-hubspot/setup.py index 46fd1ff67bfe..2d4955656504 100644 --- a/airbyte-integrations/connectors/source-hubspot/setup.py +++ b/airbyte-integrations/connectors/source-hubspot/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk~=0.1", "backoff==1.11.1", "pendulum==2.1.2", "requests==2.26.0", diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/__init__.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 From 6ab788b88c6f965aeec730967256d2146c3a7f9d Mon Sep 17 00:00:00 2001 From: auganbay Date: Mon, 7 Feb 2022 15:01:25 +0600 Subject: [PATCH 7/9] fix importing airbyte models --- .../connectors/source-hubspot/source_hubspot/source.py | 3 +-- .../connectors/source-hubspot/unit_tests/test_client.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index b6c74eaea9b9..8ec61219ce18 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -5,8 +5,7 @@ import logging from typing import Any, MutableMapping -from airbyte_cdk.models import ConfiguredAirbyteStream -from airbyte_cdk.sources.deprecated.base_source import BaseClient, BaseSource +from airbyte_cdk.sources.deprecated.base_source import BaseClient, BaseSource, ConfiguredAirbyteStream from .client import Client diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py index cc738ae78862..35d939b99b9b 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py @@ -7,7 +7,7 @@ from functools import partial import pytest -from airbyte_cdk.models import ConfiguredAirbyteCatalog, Type +from airbyte_cdk.sources.deprecated.base_source import ConfiguredAirbyteCatalog, Type from source_hubspot.api import API, PROPERTIES_PARAM_MAX_LENGTH, split_properties from source_hubspot.client import Client from source_hubspot.source import SourceHubspot From fcc8d02e17f36b698fd46bd63600a4d40c51b65e Mon Sep 17 00:00:00 2001 From: auganbay Date: Thu, 10 Feb 2022 10:19:34 +0600 Subject: [PATCH 8/9] bump the version --- airbyte-integrations/connectors/source-hubspot/Dockerfile | 2 +- docs/integrations/sources/hubspot.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile index 912650348fdf..93915b4ec0f4 100644 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot/Dockerfile @@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.38 +LABEL io.airbyte.version=0.1.39 LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index 4b9bd4f46c95..786687814a0f 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -112,6 +112,7 @@ If you are using Oauth, most of the streams require the appropriate [scopes](htt | Version | Date | Pull Request | Subject | |:--------|:-----------| :--- |:-----------------------------------------------------------------------------------------------------------------------------------------------| +| 0.1.39 | 2022-02-10 | [10055](https://github.com/airbytehq/airbyte/pull/10055) | Bug fix: reading not initialized stream | | 0.1.38 | 2022-02-03 | [9786](https://github.com/airbytehq/airbyte/pull/9786) | Add new streams for engagements(calls, emails, meetings, notes and tasks) | | 0.1.37 | 2022-01-27 | [9555](https://github.com/airbytehq/airbyte/pull/9555) | Getting form_submission for all forms | | 0.1.36 | 2022-01-22 | [7784](https://github.com/airbytehq/airbyte/pull/7784) | Add Property History Stream | From b929cf034a74aa03628ee7e5068e213aa859eaee Mon Sep 17 00:00:00 2001 From: auganbay Date: Thu, 10 Feb 2022 10:38:41 +0600 Subject: [PATCH 9/9] update spec and def yamls --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index e10d784c2250..7105c1cba5e8 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -314,7 +314,7 @@ - name: HubSpot sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c dockerRepository: airbyte/source-hubspot - dockerImageTag: 0.1.38 + dockerImageTag: 0.1.39 documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot icon: hubspot.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 9ec51abce0b6..caccc3d4dcf2 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3068,7 +3068,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-hubspot:0.1.38" +- dockerImage: "airbyte/source-hubspot:0.1.39" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot" connectionSpecification: