From 34b4821c8cc826daa928c403d233471e4f20ba6e Mon Sep 17 00:00:00 2001 From: Augustin Date: Wed, 28 Sep 2022 16:21:59 +0200 Subject: [PATCH] source-hubspot: migrate to per-stream state (#17120) --- .../main/resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-hubspot/Dockerfile | 2 +- .../source-hubspot/source_hubspot/source.py | 16 ++++++++++------ docs/integrations/sources/hubspot.md | 11 ++++++----- 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 07a3c3fd57e2..4d47d4f6aaf1 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -464,7 +464,7 @@ - name: HubSpot sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c dockerRepository: airbyte/source-hubspot - dockerImageTag: 0.2.0 + dockerImageTag: 0.2.1 documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot icon: hubspot.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 1ec358895779..a9b4314d1c7a 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -4618,7 +4618,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-hubspot:0.2.0" +- dockerImage: "airbyte/source-hubspot:0.2.1" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile index 71484d687c4b..86a7233f7bbb 100644 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot/Dockerfile @@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.version=0.2.1 LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index 9c67ccd5a4da..db286e7fd7e5 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -2,14 +2,14 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -import copy import logging -from typing import Any, Iterator, List, Mapping, MutableMapping, Optional, Tuple +from typing import Any, Iterator, List, Mapping, MutableMapping, Optional, Tuple, Union import requests from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog +from airbyte_cdk.models import AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.utils.schema_helpers import split_config from airbyte_cdk.utils.event_timing import create_timer @@ -137,17 +137,21 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: return available_streams def read( - self, logger: logging.Logger, config: Mapping[str, Any], catalog: ConfiguredAirbyteCatalog, state: MutableMapping[str, Any] = None + self, + logger: logging.Logger, + config: Mapping[str, Any], + catalog: ConfiguredAirbyteCatalog, + state: Union[List[AirbyteStateMessage], MutableMapping[str, Any]] = None, ) -> Iterator[AirbyteMessage]: """ This method is overridden to check whether the stream `quotes` exists in the source, if not skip reading that stream. """ - connector_state = copy.deepcopy(state or {}) logger.info(f"Starting syncing {self.name}") config, internal_config = split_config(config) # TODO assert all streams exist in the connector # get the streams once in case the connector needs to make any queries to generate them stream_instances = {s.name: s for s in self.streams(config)} + state_manager = ConnectorStateManager(stream_instance_map=stream_instances, state=state) self._stream_to_instance_map = stream_instances with create_timer(self.name) as timer: for configured_stream in catalog.streams: @@ -165,7 +169,7 @@ def read( logger=logger, stream_instance=stream_instance, configured_stream=configured_stream, - connector_state=connector_state, + state_manager=state_manager, internal_config=internal_config, ) except Exception as e: diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index c5d07788c1cf..e5936c70944f 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -7,7 +7,7 @@ This page guides you through setting up the HubSpot source connector. You can use OAuth, API key, or Private App to authenticate your HubSpot account. If you choose to use OAuth or Private App, you need to configure the appropriate [scopes](https://legacydocs.hubspot.com/docs/methods/oauth2/initiate-oauth-integration#scopes) for the following streams: | Stream | Required Scope | -|:----------------------------|:---------------------------------------------------------------------------------| +| :-------------------------- | :------------------------------------------------------------------------------- | | `campaigns` | `content` | | `companies` | `contacts` | | `contact_lists` | `contacts` | @@ -131,13 +131,14 @@ Now that you have set up the Hubspot source connector, check out the following H ## Changelog | Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------| +| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------- | +| 0.2.1 | 2022-09-26 | [17120](https://github.com/airbytehq/airbyte/pull/17120) | Migrate to per-stream state. | | 0.2.0 | 2022-09-13 | [16632](https://github.com/airbytehq/airbyte/pull/16632) | Remove Feedback Submissions stream as the one using unstable (beta) API. | | 0.1.83 | 2022-09-01 | [16214](https://github.com/airbytehq/airbyte/pull/16214) | Update Tickets, fix missing properties and change how state is updated. | -| 0.1.82 | 2022-08-18 | [15110](https://github.com/airbytehq/airbyte/pull/15110) | Check if it has a state on search streams before first sync | +| 0.1.82 | 2022-08-18 | [15110](https://github.com/airbytehq/airbyte/pull/15110) | Check if it has a state on search streams before first sync | | 0.1.81 | 2022-08-05 | [15354](https://github.com/airbytehq/airbyte/pull/15354) | Fix `Deals` stream schema | | 0.1.80 | 2022-08-01 | [15156](https://github.com/airbytehq/airbyte/pull/15156) | Fix 401 error while retrieving associations using OAuth | -| 0.1.79 | 2022-07-28 | [15144](https://github.com/airbytehq/airbyte/pull/15144) | Revert v0.1.78 due to permission issues | +| 0.1.79 | 2022-07-28 | [15144](https://github.com/airbytehq/airbyte/pull/15144) | Revert v0.1.78 due to permission issues | | 0.1.78 | 2022-07-28 | [15099](https://github.com/airbytehq/airbyte/pull/15099) | Fix to fetch associations when using incremental mode | | 0.1.77 | 2022-07-26 | [15035](https://github.com/airbytehq/airbyte/pull/15035) | Make PropertyHistory stream read historic data not limited to 30 days | | 0.1.76 | 2022-07-25 | [14999](https://github.com/airbytehq/airbyte/pull/14999) | Partially revert changes made in v0.1.75 | @@ -145,7 +146,7 @@ Now that you have set up the Hubspot source connector, check out the following H | 0.1.74 | 2022-07-25 | [14412](https://github.com/airbytehq/airbyte/pull/14412) | Add private app authentication | | 0.1.73 | 2022-07-13 | [14666](https://github.com/airbytehq/airbyte/pull/14666) | Decrease number of http requests made, disable Incremental mode for PropertyHistory stream | | 0.1.72 | 2022-06-24 | [14054](https://github.com/airbytehq/airbyte/pull/14054) | Extended error logging | -| 0.1.71 | 2022-06-24 | [14102](https://github.com/airbytehq/airbyte/pull/14102) | Removed legacy `AirbyteSentry` dependency from the code | +| 0.1.71 | 2022-06-24 | [14102](https://github.com/airbytehq/airbyte/pull/14102) | Removed legacy `AirbyteSentry` dependency from the code | | 0.1.70 | 2022-06-16 | [13837](https://github.com/airbytehq/airbyte/pull/13837) | Fix the missing data in CRM streams issue | | 0.1.69 | 2022-06-10 | [13691](https://github.com/airbytehq/airbyte/pull/13691) | Fix the `URI Too Long` issue | | 0.1.68 | 2022-06-08 | [13596](https://github.com/airbytehq/airbyte/pull/13596) | Fix for the `property_history` which did not emit records |