From c779c49e9b7ca7fe9b88caa7b2faac0f38f4d9d4 Mon Sep 17 00:00:00 2001 From: Brian Lai <51336873+brianjlai@users.noreply.github.com> Date: Fri, 6 Jan 2023 20:17:51 -0500 Subject: [PATCH] Allow for custom requesters to be defined in low-code manifests (#21001) * Allow for custom requesters to be defined in low-code manifests * add test for custom requester component * bump versions and changelog --- airbyte-cdk/python/CHANGELOG.md | 3 +++ .../declarative_component_schema.yaml | 21 ++++++++++++++- .../models/declarative_component_schema.py | 11 +++++++- airbyte-cdk/python/setup.py | 2 +- .../sources/declarative/external_component.py | 13 ++++++++++ .../test_manifest_declarative_source.py | 26 ++++++++++++++++++- 6 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 airbyte-cdk/python/unit_tests/sources/declarative/external_component.py diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index d45f18a2f16f..7a03b7252509 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.18.1 +Allow for CustomRequester to be defined within declarative manifests + ## 0.18.0 Adding `cursor_granularity` to the declarative API of DatetimeStreamSlicer diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index b9d9eb37a5b2..323838927a41 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -273,6 +273,23 @@ definitions: $options: type: object additionalProperties: true + CustomRequester: + description: Requester component whose behavior is derived from a custom code implementation of the connector + type: object + additionalProperties: true + required: + - type + - class_name + properties: + type: + type: string + enum: [CustomRequester] + class_name: + type: string + additionalProperties: true + $options: + type: object + additionalProperties: true CustomRetriever: description: Retriever component whose behavior is derived from a custom code implementation of the connector type: object @@ -963,7 +980,9 @@ definitions: record_selector: "$ref": "#/definitions/RecordSelector" requester: - "$ref": "#/definitions/HttpRequester" + anyOf: + - "$ref": "#/definitions/CustomRequester" + - "$ref": "#/definitions/HttpRequester" name: type: string default: "" diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 4a44dfaf4349..0611c95abb63 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -99,6 +99,15 @@ class Config: _options: Optional[Dict[str, Any]] = Field(None, alias="$options") +class CustomRequester(BaseModel): + class Config: + extra = Extra.allow + + type: Literal["CustomRequester"] + class_name: str + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + class CustomRetriever(BaseModel): class Config: extra = Extra.allow @@ -461,7 +470,7 @@ class ParentStreamConfig(BaseModel): class SimpleRetriever(BaseModel): type: Literal["SimpleRetriever"] record_selector: RecordSelector - requester: HttpRequester + requester: Union[CustomRequester, HttpRequester] name: Optional[str] = "" paginator: Optional[Union[DefaultPaginator, NoPagination]] = None primary_key: Optional[PrimaryKey] = None diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 66ce7c067620..6d19757c91f2 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.18.0", + version="0.18.1", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/external_component.py b/airbyte-cdk/python/unit_tests/sources/declarative/external_component.py new file mode 100644 index 000000000000..9f51f5f04c11 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/external_component.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.requesters import HttpRequester + + +class SampleCustomComponent(HttpRequester): + """ + A test class used to validate manifests that rely on custom defined Python components + """ + + pass diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py index b2760bb8d43c..fcdf9723488f 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py @@ -94,7 +94,31 @@ def test_valid_manifest(self): }, "record_selector": {"extractor": {"field_pointer": ["result"]}}, }, - } + }, + { + "type": "DeclarativeStream", + "$options": {"name": "stream_with_custom_requester", "primary_key": "id", "url_base": "https://api.sendgrid.com"}, + "schema_loader": { + "name": "{{ options.stream_name }}", + "file_path": "./source_sendgrid/schemas/{{ options.name }}.yaml", + }, + "retriever": { + "paginator": { + "type": "DefaultPaginator", + "page_size": 10, + "page_size_option": {"inject_into": "request_parameter", "field_name": "page_size"}, + "page_token_option": {"inject_into": "path"}, + "pagination_strategy": {"type": "CursorPagination", "cursor_value": "{{ response._metadata.next }}"}, + }, + "requester": { + "type": "CustomRequester", + "class_name": "unit_tests.sources.declarative.external_component.SampleCustomComponent", + "path": "/v3/marketing/lists", + "custom_request_parameters": {"page_size": 10}, + }, + "record_selector": {"extractor": {"field_pointer": ["result"]}}, + }, + }, ], "check": {"type": "CheckStream", "stream_names": ["lists"]}, }