CDK Lowcode: allow parametrizing initial page when using PageIncrement (

#19712) * #CDK Lowcode: allow parametrizing initial page when using PageIncrement * bump minor version
airbytehq · Nov 28, 2022 · fc52644 · fc52644
1 parent 66edbfb
commit fc52644
Show file tree

Hide file tree

Showing 5 changed files with 20 additions and 10 deletions.
diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md
@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.10.0
+Low-code: Add `start_from_page` option to a PageIncrement class
+
 ## 0.9.5
 Low-code: Add jinja macro `format_datetime`
 

diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json b/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json
@@ -1102,6 +1102,9 @@
           "properties": {
             "page_size": {
               "type": "integer"
+            },
+            "start_from_page": {
+              "type": "integer"
             }
           }
         }

diff --git a/...python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py b/...python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py
@@ -17,13 +17,15 @@ class PageIncrement(PaginationStrategy, JsonSchemaMixin):
 
     Attributes:
         page_size (int): the number of records to request
+        start_from_page (int): number of the initial page
     """
 
     page_size: int
     options: InitVar[Mapping[str, Any]]
+    start_from_page: int = 0
 
     def __post_init__(self, options: Mapping[str, Any]):
-        self._page = 0
+        self._page = self.start_from_page
 
     def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]:
         if len(last_records) < self.page_size:
@@ -33,7 +35,7 @@ def next_page_token(self, response: requests.Response, last_records: List[Mappin
             return self._page
 
     def reset(self):
-        self._page = 0
+        self._page = self.start_from_page
 
     def get_page_size(self) -> Optional[int]:
         return self.page_size
diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py
@@ -15,7 +15,7 @@
 
 setup(
     name="airbyte-cdk",
-    version="0.9.5",
+    version="0.10.0",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

diff --git a/...te-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py b/...te-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py
@@ -10,15 +10,17 @@
 
 
 @pytest.mark.parametrize(
-    "test_name, page_size, expected_next_page_token, expected_offset",
+    "test_name, page_size, start_from, expected_next_page_token, expected_offset",
     [
-        ("test_same_page_size", 2, 1, 1),
-        ("test_larger_page_size", 3, None, 0),
+        ("test_same_page_size_start_from_0", 2, 1, 2, 2),
+        ("test_larger_page_size_start_from_0", 3, 1, None, 1),
+        ("test_same_page_size_start_from_1", 2, 0, 1, 1),
+        ("test_larger_page_size_start_from_0", 3, 0, None, 0)
     ],
 )
-def test_page_increment_paginator_strategy(test_name, page_size, expected_next_page_token, expected_offset):
-    paginator_strategy = PageIncrement(page_size, options={})
-    assert paginator_strategy._page == 0
+def test_page_increment_paginator_strategy(test_name, page_size, start_from, expected_next_page_token, expected_offset):
+    paginator_strategy = PageIncrement(page_size, options={}, start_from_page=start_from)
+    assert paginator_strategy._page == start_from
 
     response = requests.Response()
 
@@ -32,4 +34,4 @@ def test_page_increment_paginator_strategy(test_name, page_size, expected_next_p
     assert expected_offset == paginator_strategy._page
 
     paginator_strategy.reset()
-    assert 0 == paginator_strategy._page
+    assert start_from == paginator_strategy._page