diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index d45f18a2f16f9..7a03b7252509a 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.18.1 +Allow for CustomRequester to be defined within declarative manifests + ## 0.18.0 Adding `cursor_granularity` to the declarative API of DatetimeStreamSlicer diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index b9d9eb37a5b21..323838927a410 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -273,6 +273,23 @@ definitions: $options: type: object additionalProperties: true + CustomRequester: + description: Requester component whose behavior is derived from a custom code implementation of the connector + type: object + additionalProperties: true + required: + - type + - class_name + properties: + type: + type: string + enum: [CustomRequester] + class_name: + type: string + additionalProperties: true + $options: + type: object + additionalProperties: true CustomRetriever: description: Retriever component whose behavior is derived from a custom code implementation of the connector type: object @@ -963,7 +980,9 @@ definitions: record_selector: "$ref": "#/definitions/RecordSelector" requester: - "$ref": "#/definitions/HttpRequester" + anyOf: + - "$ref": "#/definitions/CustomRequester" + - "$ref": "#/definitions/HttpRequester" name: type: string default: "" diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 4a44dfaf43492..0611c95abb637 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -99,6 +99,15 @@ class Config: _options: Optional[Dict[str, Any]] = Field(None, alias="$options") +class CustomRequester(BaseModel): + class Config: + extra = Extra.allow + + type: Literal["CustomRequester"] + class_name: str + _options: Optional[Dict[str, Any]] = Field(None, alias="$options") + + class CustomRetriever(BaseModel): class Config: extra = Extra.allow @@ -461,7 +470,7 @@ class ParentStreamConfig(BaseModel): class SimpleRetriever(BaseModel): type: Literal["SimpleRetriever"] record_selector: RecordSelector - requester: HttpRequester + requester: Union[CustomRequester, HttpRequester] name: Optional[str] = "" paginator: Optional[Union[DefaultPaginator, NoPagination]] = None primary_key: Optional[PrimaryKey] = None diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 66ce7c067620e..6d19757c91f28 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.18.0", + version="0.18.1", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/external_component.py b/airbyte-cdk/python/unit_tests/sources/declarative/external_component.py new file mode 100644 index 0000000000000..9f51f5f04c110 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/external_component.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.requesters import HttpRequester + + +class SampleCustomComponent(HttpRequester): + """ + A test class used to validate manifests that rely on custom defined Python components + """ + + pass diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py index b2760bb8d43cd..fcdf9723488f4 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py @@ -94,7 +94,31 @@ def test_valid_manifest(self): }, "record_selector": {"extractor": {"field_pointer": ["result"]}}, }, - } + }, + { + "type": "DeclarativeStream", + "$options": {"name": "stream_with_custom_requester", "primary_key": "id", "url_base": "https://api.sendgrid.com"}, + "schema_loader": { + "name": "{{ options.stream_name }}", + "file_path": "./source_sendgrid/schemas/{{ options.name }}.yaml", + }, + "retriever": { + "paginator": { + "type": "DefaultPaginator", + "page_size": 10, + "page_size_option": {"inject_into": "request_parameter", "field_name": "page_size"}, + "page_token_option": {"inject_into": "path"}, + "pagination_strategy": {"type": "CursorPagination", "cursor_value": "{{ response._metadata.next }}"}, + }, + "requester": { + "type": "CustomRequester", + "class_name": "unit_tests.sources.declarative.external_component.SampleCustomComponent", + "path": "/v3/marketing/lists", + "custom_request_parameters": {"page_size": 10}, + }, + "record_selector": {"extractor": {"field_pointer": ["result"]}}, + }, + }, ], "check": {"type": "CheckStream", "stream_names": ["lists"]}, }