From 06a01b57c0d7d0ef16c5f37cb0b9c3e7b9e6ea68 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Mon, 3 Oct 2022 17:02:41 -0700 Subject: [PATCH] [#17107] Fix propagation of $options field in arrays (#17196) * input can only be a string * remove debug print * cleanup tests * fix in the factory * Add a unit test * fix for lists * Update changelog * Update changelog --- airbyte-cdk/python/CHANGELOG.md | 4 ++ .../sources/declarative/parsers/factory.py | 4 +- airbyte-cdk/python/setup.py | 2 +- .../sources/declarative/test_factory.py | 60 +++++++++++++++++-- 4 files changed, 60 insertions(+), 10 deletions(-) diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index ef00c23bd5c9..af039710854b 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.1.92 +- Low-code: Properly propagate $options to array items +- Low-code: Log request and response when running check operation in debug mode + ## 0.1.91 - Low-code: Rename LimitPaginator to DefaultPaginator and move page_size field to PaginationStrategy diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py index d31a7d5896b8..676074d174b3 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py @@ -138,13 +138,11 @@ def build(self, class_or_class_name: Union[str, Type], config, instantiate: bool class_ = self._get_class_from_fully_qualified_class_name(class_or_class_name) else: class_ = class_or_class_name - # create components in options before propagating them if OPTIONS_STR in kwargs: kwargs[OPTIONS_STR] = { k: self._create_subcomponent(k, v, kwargs, config, class_, instantiate) for k, v in kwargs[OPTIONS_STR].items() } - updated_kwargs = {k: self._create_subcomponent(k, v, kwargs, config, class_, instantiate) for k, v in kwargs.items()} if instantiate: @@ -216,7 +214,7 @@ def _create_subcomponent(self, key, definition, kwargs, config, parent_class, in self._create_subcomponent( key, sub, - self._merge_dicts(kwargs.get(OPTIONS_STR, dict()), self._get_subcomponent_options(sub)), + kwargs, config, parent_class, instantiate, diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 2b833673c5a5..e0c3c6ab9ed5 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.1.91", + version="0.1.92", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py index e103aaaea85b..7cf63571b5ad 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py @@ -328,7 +328,7 @@ def test_full_config(): primary_key: "id" extractor: $ref: "*ref(extractor)" - field_pointer: ["result"] + field_pointer: ["{{ options['name'] }}"] retriever: $ref: "*ref(retriever)" requester: @@ -365,7 +365,7 @@ def test_full_config(): assert type(stream.retriever.record_selector) == RecordSelector assert type(stream.retriever.record_selector.extractor.decoder) == JsonDecoder - assert [fp.eval(input_config) for fp in stream.retriever.record_selector.extractor.field_pointer] == ["result"] + assert [fp.eval(input_config) for fp in stream.retriever.record_selector.extractor.field_pointer] == ["lists"] assert type(stream.retriever.record_selector.record_filter) == RecordFilter assert stream.retriever.record_selector.record_filter._filter_interpolator.condition == "{{ record['id'] > stream_state['id'] }}" assert stream.schema_loader._get_json_filepath() == "./source_sendgrid/schemas/lists.json" @@ -378,18 +378,24 @@ def test_full_config(): assert stream.retriever.requester.path.default == "marketing/lists" -def test_create_record_selector(): - content = """ +@pytest.mark.parametrize( + "test_name, record_selector, expected_runtime_selector", + [("test_static_record_selector", "result", "result"), ("test_options_record_selector", "{{ options['name'] }}", "lists")], +) +def test_create_record_selector(test_name, record_selector, expected_runtime_selector): + content = f""" extractor: type: DpathExtractor selector: + $options: + name: "lists" class_name: airbyte_cdk.sources.declarative.extractors.record_selector.RecordSelector record_filter: class_name: airbyte_cdk.sources.declarative.extractors.record_filter.RecordFilter condition: "{{ record['id'] > stream_state['id'] }}" extractor: $ref: "*ref(extractor)" - field_pointer: ["result"] + field_pointer: ["{record_selector}"] """ config = parser.parse(content) @@ -398,10 +404,52 @@ def test_create_record_selector(): selector = factory.create_component(config["selector"], input_config)() assert isinstance(selector, RecordSelector) assert isinstance(selector.extractor, DpathExtractor) - assert [fp.eval(input_config) for fp in selector.extractor.field_pointer] == ["result"] + assert [fp.eval(input_config) for fp in selector.extractor.field_pointer] == [expected_runtime_selector] assert isinstance(selector.record_filter, RecordFilter) +@pytest.mark.parametrize( + "test_name, content, expected_field_pointer_value", + [ + ( + "test_option_in_selector", + """ + extractor: + type: DpathExtractor + field_pointer: ["{{ options['name'] }}"] + selector: + class_name: airbyte_cdk.sources.declarative.extractors.record_selector.RecordSelector + $options: + name: "selector" + extractor: "*ref(extractor)" + """, + "selector", + ), + ( + "test_option_in_extractor", + """ + extractor: + type: DpathExtractor + $options: + name: "extractor" + field_pointer: ["{{ options['name'] }}"] + selector: + class_name: airbyte_cdk.sources.declarative.extractors.record_selector.RecordSelector + $options: + name: "selector" + extractor: "*ref(extractor)" + """, + "extractor", + ), + ], +) +def test_options_propagation(test_name, content, expected_field_pointer_value): + config = parser.parse(content) + + selector = factory.create_component(config["selector"], input_config, True)() + assert selector.extractor.field_pointer[0].eval(input_config) == expected_field_pointer_value + + def test_create_requester(): content = """ requester: