From e9e37a744d3a805c36ae84363b6105e1aab05298 Mon Sep 17 00:00:00 2001 From: Roman Yermilov Date: Mon, 21 Nov 2022 14:29:46 +0400 Subject: [PATCH] Source Close.com: avoid custom pagination component --- .../source_close_com/__init__.py | 3 +- .../source_close_com/close_com.yaml | 3 +- .../offset_increment_workaround.py | 41 ----------- .../unit_tests/offset_increment_unit_test.py | 72 ------------------- 4 files changed, 2 insertions(+), 117 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-close-com/source_close_com/offset_increment_workaround.py delete mode 100644 airbyte-integrations/connectors/source-close-com/unit_tests/offset_increment_unit_test.py diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/__init__.py b/airbyte-integrations/connectors/source-close-com/source_close_com/__init__.py index 69a2e81420f0..a127dfa6f4ac 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/__init__.py +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/__init__.py @@ -23,6 +23,5 @@ from .source_lc import SourceCloseCom from .single_stream_slicer import SingleStreamSlicer -from .offset_increment_workaround import OffsetIncrementWorkaround -__all__ = ["OffsetIncrementWorkaround", "SourceCloseCom", "SingleStreamSlicer"] +__all__ = ["SourceCloseCom", "SingleStreamSlicer"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/close_com.yaml b/airbyte-integrations/connectors/source-close-com/source_close_com/close_com.yaml index 116d567f0657..1035150fc313 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/close_com.yaml +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/close_com.yaml @@ -26,8 +26,7 @@ definitions: inject_into: "request_parameter" field_name: "_skip" pagination_strategy: - class_name: "source_close_com.OffsetIncrementWorkaround" -# type: OffsetIncrement + type: OffsetIncrement page_size: "{{ options['items_per_page'] }}" cursor_paginator: type: DefaultPaginator diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/offset_increment_workaround.py b/airbyte-integrations/connectors/source-close-com/source_close_com/offset_increment_workaround.py deleted file mode 100644 index 8f1f8eb22979..000000000000 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/offset_increment_workaround.py +++ /dev/null @@ -1,41 +0,0 @@ -from dataclasses import InitVar, dataclass -from typing import Any, List, Mapping, Optional, Union - -import requests -from airbyte_cdk.sources.declarative.interpolation import InterpolatedString -from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import PaginationStrategy -from airbyte_cdk.sources.declarative.types import Config -from dataclasses_jsonschema import JsonSchemaMixin - - -@dataclass -class OffsetIncrementWorkaround(PaginationStrategy, JsonSchemaMixin): - """ - Pagination strategy that returns the number of records reads so far and returns it as the next page token - This workaround makes `page_size` to be Interpolated optional string - - Attributes: - page_size Union[InterpolatedString, str]: the number of records to request - """ - - config: Config - - page_size: Union[InterpolatedString, str] - options: InitVar[Mapping[str, Any]] - - def __post_init__(self, options: Mapping[str, Any]): - self._offset = 0 - self.page_size = InterpolatedString.create(self.page_size, options=options) - - def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: - if len(last_records) < self.page_size.eval(self.config): - return None - else: - self._offset += len(last_records) - return self._offset - - def reset(self): - self._offset = 0 - - def get_page_size(self) -> Optional[int]: - return self.page_size.eval(self.config) diff --git a/airbyte-integrations/connectors/source-close-com/unit_tests/offset_increment_unit_test.py b/airbyte-integrations/connectors/source-close-com/unit_tests/offset_increment_unit_test.py deleted file mode 100644 index ca5ea0a9a361..000000000000 --- a/airbyte-integrations/connectors/source-close-com/unit_tests/offset_increment_unit_test.py +++ /dev/null @@ -1,72 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# -from source_close_com import OffsetIncrementWorkaround - - -def test_next_page_token_to_be_interpolated(mocker): - offset_increment = OffsetIncrementWorkaround( - config=mocker.MagicMock(), - page_size="{{ options['optionally_defined_page_size'] }}", - options={"optionally_defined_page_size": 1} - ) - assert offset_increment.get_page_size() == 1 - - -def test_next_page_token_has_more_records_flow(mocker): - offset_increment = OffsetIncrementWorkaround( - config=mocker.MagicMock(), - page_size="{{ options['optionally_defined_page_size'] }}", - options={"optionally_defined_page_size": 1} - ) - - assert getattr(offset_increment, "_offset") == 0 - - assert offset_increment.next_page_token( - response=mocker.MagicMock(), - last_records=[{}] - ) == 1 - - assert getattr(offset_increment, "_offset") == 1 - - assert offset_increment.next_page_token( - response=mocker.MagicMock(), - last_records=[{}] - ) == 2 - - assert getattr(offset_increment, "_offset") == 2 - - -def test_next_page_token_last_page(mocker): - offset_increment = OffsetIncrementWorkaround( - config=mocker.MagicMock(), - page_size="{{ options['optionally_defined_page_size'] }}", - options={"optionally_defined_page_size": 2} - ) - setattr(offset_increment, "_offset", 2) - - assert offset_increment.next_page_token( - response=mocker.MagicMock(), - last_records=[{}] - ) is None - - -def test_next_page_token_reset(mocker): - offset_increment = OffsetIncrementWorkaround( - config=mocker.MagicMock(), - page_size="{{ options['optionally_defined_page_size'] }}", - options={"optionally_defined_page_size": 1} - ) - - assert getattr(offset_increment, "_offset") == 0 - - offset_increment.next_page_token( - response=mocker.MagicMock(), - last_records=[{}] - ) - - assert getattr(offset_increment, "_offset") == 1 - - offset_increment.reset() - - assert getattr(offset_increment, "_offset") == 0