Allow for custom requesters to be defined in low-code manifests (#21001)

* Allow for custom requesters to be defined in low-code manifests * add test for custom requester component * bump versions and changelog
airbytehq · Jan 13, 2023 · c779c49 · c779c49
1 parent 972794a
commit c779c49
Show file tree

Hide file tree

Showing 6 changed files with 72 additions and 4 deletions.
diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md
@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.18.1
+Allow for CustomRequester to be defined within declarative manifests
+
 ## 0.18.0
 Adding `cursor_granularity` to the declarative API of DatetimeStreamSlicer
 

diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml
@@ -273,6 +273,23 @@ definitions:
       $options:
         type: object
         additionalProperties: true
+  CustomRequester:
+    description: Requester component whose behavior is derived from a custom code implementation of the connector
+    type: object
+    additionalProperties: true
+    required:
+      - type
+      - class_name
+    properties:
+      type:
+        type: string
+        enum: [CustomRequester]
+      class_name:
+        type: string
+        additionalProperties: true
+      $options:
+        type: object
+        additionalProperties: true
   CustomRetriever:
     description: Retriever component whose behavior is derived from a custom code implementation of the connector
     type: object
@@ -963,7 +980,9 @@ definitions:
       record_selector:
         "$ref": "#/definitions/RecordSelector"
       requester:
-        "$ref": "#/definitions/HttpRequester"
+        anyOf:
+          - "$ref": "#/definitions/CustomRequester"
+          - "$ref": "#/definitions/HttpRequester"
       name:
         type: string
         default: ""

diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py
@@ -99,6 +99,15 @@ class Config:
     _options: Optional[Dict[str, Any]] = Field(None, alias="$options")
 
 
+class CustomRequester(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal["CustomRequester"]
+    class_name: str
+    _options: Optional[Dict[str, Any]] = Field(None, alias="$options")
+
+
 class CustomRetriever(BaseModel):
     class Config:
         extra = Extra.allow
@@ -461,7 +470,7 @@ class ParentStreamConfig(BaseModel):
 class SimpleRetriever(BaseModel):
     type: Literal["SimpleRetriever"]
     record_selector: RecordSelector
-    requester: HttpRequester
+    requester: Union[CustomRequester, HttpRequester]
     name: Optional[str] = ""
     paginator: Optional[Union[DefaultPaginator, NoPagination]] = None
     primary_key: Optional[PrimaryKey] = None

diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py
@@ -15,7 +15,7 @@
 
 setup(
     name="airbyte-cdk",
-    version="0.18.0",
+    version="0.18.1",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/external_component.py b/airbyte-cdk/python/unit_tests/sources/declarative/external_component.py
@@ -0,0 +1,13 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+from airbyte_cdk.sources.declarative.requesters import HttpRequester
+
+
+class SampleCustomComponent(HttpRequester):
+    """
+    A test class used to validate manifests that rely on custom defined Python components
+    """
+
+    pass
diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py
@@ -94,7 +94,31 @@ def test_valid_manifest(self):
                         },
                         "record_selector": {"extractor": {"field_pointer": ["result"]}},
                     },
-                }
+                },
+                {
+                    "type": "DeclarativeStream",
+                    "$options": {"name": "stream_with_custom_requester", "primary_key": "id", "url_base": "https://api.sendgrid.com"},
+                    "schema_loader": {
+                        "name": "{{ options.stream_name }}",
+                        "file_path": "./source_sendgrid/schemas/{{ options.name }}.yaml",
+                    },
+                    "retriever": {
+                        "paginator": {
+                            "type": "DefaultPaginator",
+                            "page_size": 10,
+                            "page_size_option": {"inject_into": "request_parameter", "field_name": "page_size"},
+                            "page_token_option": {"inject_into": "path"},
+                            "pagination_strategy": {"type": "CursorPagination", "cursor_value": "{{ response._metadata.next }}"},
+                        },
+                        "requester": {
+                            "type": "CustomRequester",
+                            "class_name": "unit_tests.sources.declarative.external_component.SampleCustomComponent",
+                            "path": "/v3/marketing/lists",
+                            "custom_request_parameters": {"page_size": 10},
+                        },
+                        "record_selector": {"extractor": {"field_pointer": ["result"]}},
+                    },
+                },
             ],
             "check": {"type": "CheckStream", "stream_names": ["lists"]},
         }