diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index b37ad9f9520c..2d011511854f 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.10.0 +Low-code: Add `start_from_page` option to a PageIncrement class + ## 0.9.5 Low-code: Add jinja macro `format_datetime` diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json b/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json index 8076c6f4ad7f..4d3341b8e2ea 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json @@ -1102,6 +1102,9 @@ "properties": { "page_size": { "type": "integer" + }, + "start_from_page": { + "type": "integer" } } } diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py index 530c8fbcfcc7..7f3ffaf207b5 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py @@ -17,13 +17,15 @@ class PageIncrement(PaginationStrategy, JsonSchemaMixin): Attributes: page_size (int): the number of records to request + start_from_page (int): number of the initial page """ page_size: int options: InitVar[Mapping[str, Any]] + start_from_page: int = 0 def __post_init__(self, options: Mapping[str, Any]): - self._page = 0 + self._page = self.start_from_page def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: if len(last_records) < self.page_size: @@ -33,7 +35,7 @@ def next_page_token(self, response: requests.Response, last_records: List[Mappin return self._page def reset(self): - self._page = 0 + self._page = self.start_from_page def get_page_size(self) -> Optional[int]: return self.page_size diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 11222c1020ab..133323cb4827 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.9.5", + version="0.10.0", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py index 9d85cf8298b9..699e0f794a50 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py @@ -10,15 +10,17 @@ @pytest.mark.parametrize( - "test_name, page_size, expected_next_page_token, expected_offset", + "test_name, page_size, start_from, expected_next_page_token, expected_offset", [ - ("test_same_page_size", 2, 1, 1), - ("test_larger_page_size", 3, None, 0), + ("test_same_page_size_start_from_0", 2, 1, 2, 2), + ("test_larger_page_size_start_from_0", 3, 1, None, 1), + ("test_same_page_size_start_from_1", 2, 0, 1, 1), + ("test_larger_page_size_start_from_0", 3, 0, None, 0) ], ) -def test_page_increment_paginator_strategy(test_name, page_size, expected_next_page_token, expected_offset): - paginator_strategy = PageIncrement(page_size, options={}) - assert paginator_strategy._page == 0 +def test_page_increment_paginator_strategy(test_name, page_size, start_from, expected_next_page_token, expected_offset): + paginator_strategy = PageIncrement(page_size, options={}, start_from_page=start_from) + assert paginator_strategy._page == start_from response = requests.Response() @@ -32,4 +34,4 @@ def test_page_increment_paginator_strategy(test_name, page_size, expected_next_p assert expected_offset == paginator_strategy._page paginator_strategy.reset() - assert 0 == paginator_strategy._page + assert start_from == paginator_strategy._page