From 423cb8bdbc2f234cf7315fb39442eb684918231b Mon Sep 17 00:00:00 2001 From: Maxime Carbonneau-Leclerc Date: Tue, 10 Jan 2023 10:03:35 -0500 Subject: [PATCH] [ISSUE-20322] updating tutorial documentation (#21163) * [ISSUE-20322] updating tutorial documentation * [ISSUE-20322] updating test and improving docs --- .../declarative/stream_slicers/datetime_stream_slicer.py | 2 +- .../python/unit_tests/sources/declarative/test_factory.py | 2 +- .../config-based/tutorial/5-incremental-reads.md | 6 ++++-- .../understanding-the-yaml-file/request-options.md | 2 +- .../understanding-the-yaml-file/stream-slicers.md | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py index 9b475f8223799..78eca8510c4cb 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py @@ -123,7 +123,7 @@ def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> """ Partition the daterange into slices of size = step. - The start of the window is the minimum datetime between start_datetime - looback_window and the stream_state's datetime + The start of the window is the minimum datetime between start_datetime - lookback_window and the stream_state's datetime The end of the window is the minimum datetime between the start of the window and end_datetime. :param sync_mode: diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py index 6265b0e638f03..d49fb43177abf 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py @@ -792,7 +792,7 @@ def test_validation_type_missing_required_fields(): min_datetime: "{{ config['start_time'] + day_delta(2) }}" end_datetime: "{{ config['end_time'] }}" cursor_field: "created" - lookback_window: "5d" + lookback_window: "P5D" start_time_option: inject_into: request_parameter field_name: created[gte] diff --git a/docs/connector-development/config-based/tutorial/5-incremental-reads.md b/docs/connector-development/config-based/tutorial/5-incremental-reads.md index dfabeff150d34..8ca82d8aa59cd 100644 --- a/docs/connector-development/config-based/tutorial/5-incremental-reads.md +++ b/docs/connector-development/config-based/tutorial/5-incremental-reads.md @@ -104,8 +104,9 @@ definitions: end_datetime: datetime: "{{ now_utc() }}" datetime_format: "%Y-%m-%d %H:%M:%S.%f+00:00" - step: "1d" + step: "P1D" datetime_format: "%Y-%m-%d" + cursor_granularity: "P1D" cursor_field: "{{ options['stream_cursor_field'] }}" ``` @@ -182,8 +183,9 @@ definitions: end_datetime: datetime: "{{ now_utc() }}" datetime_format: "%Y-%m-%d %H:%M:%S.%f+00:00" - step: "1d" + step: "P1D" datetime_format: "%Y-%m-%d" + cursor_granularity: "P1D" cursor_field: "{{ options['stream_cursor_field'] }}" retriever: record_selector: diff --git a/docs/connector-development/config-based/understanding-the-yaml-file/request-options.md b/docs/connector-development/config-based/understanding-the-yaml-file/request-options.md index b27b53e48250d..969d1e0d815df 100644 --- a/docs/connector-development/config-based/understanding-the-yaml-file/request-options.md +++ b/docs/connector-development/config-based/understanding-the-yaml-file/request-options.md @@ -131,7 +131,7 @@ The following example will set the "created[gte]" request parameter value to the stream_slicer: start_datetime: "2021-02-01T00:00:00.000000+0000", end_datetime: "2021-03-01T00:00:00.000000+0000", - step: "1d" + step: "P1D" start_time_option: field_name: "created[gte]" inject_into: "request_parameter" diff --git a/docs/connector-development/config-based/understanding-the-yaml-file/stream-slicers.md b/docs/connector-development/config-based/understanding-the-yaml-file/stream-slicers.md index 9adfe6922ec56..98a9d69ca88cc 100644 --- a/docs/connector-development/config-based/understanding-the-yaml-file/stream-slicers.md +++ b/docs/connector-development/config-based/understanding-the-yaml-file/stream-slicers.md @@ -111,7 +111,7 @@ stream_slicer: step: "P1D" ``` -will create one slice per day for the interval `2021-02-01` - `2021-03-01`. The first slice will start from the `start_datetime` and end at `start_datetime + step - granularity` like this: `{"start_time": "2021-02-01T00:00:00.000000+0000", "end_time": "2021-02-01T23:59:59.999999+0000"}`. +will create one slice per day for the interval `2021-02-01` - `2021-03-01`. The first slice will start from the `start_datetime` and end at `start_datetime + step - cursor_granularity` like this: `{"start_time": "2021-02-01T00:00:00.000000+0000", "end_time": "2021-02-01T23:59:59.999999+0000"}`. `cursor_granularity` allows the DatetimeStreamSlicer to create non overlapping slices so that the `end_time` of a slice is just before the `start_time` of the next one. The `DatetimeStreamSlicer` also supports an optional lookback window, specifying how many days before the start_datetime to read data for.