Artem1205/lowcode cdk authenticator schemaloader (#19718)

* CDK low-code: Add token_expiry_date_format to OAuth Authenticator

* CDK low-code: Resolve ref schema

* CDK low-code: Resolve ref schema

* CDK low-code: Resolve ref schema

* CDK low-code: Add test for token_expiry_date_format

* CDK low-code: set initial time before refresh request

* CDK low-code: Add test schema loader

* CDK low-code: Add test dependencies

* CDK low-code: Refactor JsonFileSchemaLoader (inherit from original CDK)

* CDK low-code: Fix SingleUseRefreshTokenOauth2Authenticator (add token_expiry_date_format)

* CDK low-code: Fix tests

* CDK low-code: Refactor import

* CDK low-code: Refactor JsonFileSchemaLoader

* CDK low-code: format

airbyte-cdk/python/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.14.0
+Low-code: Add token_expiry_date_format to OAuth Authenticator. Resolve ref schema
+
 ## 0.13.3
 Fixed `StopIteration` exception for empty streams while `check_availability` runs.
 

airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py

@@ -30,6 +30,7 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
         config (Mapping[str, Any]): The user-provided configuration as specified by the source's spec
         scopes (Optional[List[str]]): The scopes to request
         token_expiry_date (Optional[Union[InterpolatedString, str]]): The access token expiration date
+        token_expiry_date_format str: format of the datetime; provide it if expires_in is returned in datetime instead of seconds
         refresh_request_body (Optional[Mapping[str, Any]]): The request body to send in the refresh request
         grant_type: The grant_type to request for access_token
     """
@@ -43,6 +44,7 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
     scopes: Optional[List[str]] = None
     token_expiry_date: Optional[Union[InterpolatedString, str]] = None
     _token_expiry_date: pendulum.DateTime = field(init=False, repr=False, default=None)
+    token_expiry_date_format: str = None
     access_token_name: Union[InterpolatedString, str] = "access_token"
     expires_in_name: Union[InterpolatedString, str] = "expires_in"
     refresh_request_body: Optional[Mapping[str, Any]] = None
@@ -94,8 +96,11 @@ def get_refresh_request_body(self) -> Mapping[str, Any]:
     def get_token_expiry_date(self) -> pendulum.DateTime:
         return self._token_expiry_date
 
-    def set_token_expiry_date(self, value: pendulum.DateTime):
-        self._token_expiry_date = value
+    def set_token_expiry_date(self, initial_time: pendulum.DateTime, value: Union[str, int]):
+        if self.token_expiry_date_format:
+            self._token_expiry_date = pendulum.from_format(value, self.token_expiry_date_format)
+        else:
+            self._token_expiry_date = initial_time.add(seconds=value)
 
     @property
     def access_token(self) -> str:

airbyte-cdk/python/airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py

@@ -11,6 +11,7 @@
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader
 from airbyte_cdk.sources.declarative.types import Config
+from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader
 from dataclasses_jsonschema import JsonSchemaMixin
 
 
@@ -30,7 +31,7 @@ def _default_file_path() -> str:
 
 
 @dataclass
-class JsonFileSchemaLoader(SchemaLoader, JsonSchemaMixin):
+class JsonFileSchemaLoader(ResourceSchemaLoader, SchemaLoader, JsonSchemaMixin):
     """
     Loads the schema from a json file
 
@@ -63,7 +64,8 @@ def get_json_schema(self) -> Mapping[str, Any]:
             raw_schema = json.loads(raw_json_file)
         except ValueError as err:
             raise RuntimeError(f"Invalid JSON file format for file {json_schema_path}") from err
-        return raw_schema
+        self.package_name = resource
+        return self._resolve_schema_references(raw_schema)
 
     def _get_json_filepath(self):
         return self.file_path.eval(self.config)

airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py

@@ -3,7 +3,7 @@
 #
 
 from abc import abstractmethod
-from typing import Any, List, Mapping, MutableMapping, Tuple
+from typing import Any, List, Mapping, MutableMapping, Tuple, Union
 
 import pendulum
 import requests
@@ -29,10 +29,10 @@ def get_auth_header(self) -> Mapping[str, Any]:
     def get_access_token(self) -> str:
         """Returns the access token"""
         if self.token_has_expired():
-            t0 = pendulum.now()
+            current_datetime = pendulum.now()
             token, expires_in = self.refresh_access_token()
             self.access_token = token
-            self.set_token_expiry_date(t0.add(seconds=expires_in))
+            self.set_token_expiry_date(current_datetime, expires_in)
 
         return self.access_token
 
@@ -102,11 +102,11 @@ def get_scopes(self) -> List[str]:
         """List of requested scopes"""
 
     @abstractmethod
-    def get_token_expiry_date(self) -> pendulum.datetime:
+    def get_token_expiry_date(self) -> pendulum.DateTime:
         """Expiration date of the access token"""
 
     @abstractmethod
-    def set_token_expiry_date(self, value: pendulum.datetime):
+    def set_token_expiry_date(self, initial_time: pendulum.DateTime, value: Union[str, int]):
         """Setter for access token expiration date"""
 
     @abstractmethod

airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py

@@ -2,7 +2,7 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
-from typing import Any, List, Mapping, Sequence, Tuple
+from typing import Any, List, Mapping, Sequence, Tuple, Union
 
 import dpath
 import pendulum
@@ -25,6 +25,7 @@ def __init__(
         refresh_token: str,
         scopes: List[str] = None,
         token_expiry_date: pendulum.DateTime = None,
+        token_expiry_date_format: str = None,
         access_token_name: str = "access_token",
         expires_in_name: str = "expires_in",
         refresh_request_body: Mapping[str, Any] = None,
@@ -41,6 +42,7 @@ def __init__(
         self._grant_type = grant_type
 
         self._token_expiry_date = token_expiry_date or pendulum.now().subtract(days=1)
+        self._token_expiry_date_format = token_expiry_date_format
         self._access_token = None
 
     def get_token_refresh_endpoint(self) -> str:
@@ -73,8 +75,11 @@ def get_grant_type(self) -> str:
     def get_token_expiry_date(self) -> pendulum.DateTime:
         return self._token_expiry_date
 
-    def set_token_expiry_date(self, value: pendulum.DateTime):
-        self._token_expiry_date = value
+    def set_token_expiry_date(self, initial_time: pendulum.DateTime, value: Union[str, int]):
+        if self._token_expiry_date_format:
+            self._token_expiry_date = pendulum.from_format(value, self._token_expiry_date_format)
+        else:
+            self._token_expiry_date = initial_time.add(seconds=value)
 
     @property
     def access_token(self) -> str:
@@ -100,6 +105,7 @@ def __init__(
         token_refresh_endpoint: str,
         scopes: List[str] = None,
         token_expiry_date: pendulum.DateTime = None,
+        token_expiry_date_format: str = None,
         access_token_name: str = "access_token",
         expires_in_name: str = "expires_in",
         refresh_token_name: str = "refresh_token",
@@ -138,6 +144,7 @@ def __init__(
             self.get_refresh_token(),
             scopes,
             token_expiry_date,
+            token_expiry_date_format,
             access_token_name,
             expires_in_name,
             refresh_request_body,
@@ -192,7 +199,7 @@ def get_access_token(self) -> str:
             t0 = pendulum.now()
             new_access_token, access_token_expires_in, new_refresh_token = self.refresh_access_token()
             self.access_token = new_access_token
-            self.set_token_expiry_date(t0.add(seconds=access_token_expires_in))
+            self.set_token_expiry_date(t0, access_token_expires_in)
             self.set_refresh_token(new_refresh_token)
         return self.access_token
 

airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py

@@ -129,9 +129,9 @@ def get_schema(self, name: str) -> dict:
         except ValueError as err:
             raise RuntimeError(f"Invalid JSON file format for file {schema_filename}") from err
 
-        return self.__resolve_schema_references(raw_schema)
+        return self._resolve_schema_references(raw_schema)
 
-    def __resolve_schema_references(self, raw_schema: dict) -> dict:
+    def _resolve_schema_references(self, raw_schema: dict) -> dict:
         """
         Resolve links to external references and move it to local "definitions" map.
 

airbyte-cdk/python/setup.py

@@ -15,7 +15,7 @@
 
 setup(
     name="airbyte-cdk",
-    version="0.13.3",
+    version="0.14.0",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",
@@ -63,6 +63,7 @@
     python_requires=">=3.9",
     extras_require={
         "dev": [
+            "freezegun",
             "MyPy~=0.812",
             "pytest",
             "pytest-cov",

airbyte-cdk/python/unit_tests/sources/declarative/auth/test_oauth.py

@@ -4,7 +4,9 @@
 
 import logging
 
+import freezegun
 import pendulum
+import pytest
 import requests
 from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator
 from requests import Response
@@ -90,6 +92,44 @@ def test_refresh_access_token(self, mocker):
 
         assert ("access_token", 1000) == token
 
+    @pytest.mark.parametrize(
+        "expires_in_response, token_expiry_date_format",
+        [
+            (86400, None),
+            ("2020-01-02T00:00:00Z", "YYYY-MM-DDTHH:mm:ss[Z]"),
+            ('2020-01-02T00:00:00.000000+00:00', "YYYY-MM-DDTHH:mm:ss.SSSSSSZ"),
+            ("2020-01-02", "YYYY-MM-DD"),
+        ],
+        ids=["time_in_seconds", "rfc3339", "iso8601", "simple_date"]
+    )
+    @freezegun.freeze_time("2020-01-01")
+    def test_refresh_access_token_expire_format(self, mocker, expires_in_response, token_expiry_date_format):
+        next_day = "2020-01-02T00:00:00Z"
+        config.update({"token_expiry_date": pendulum.parse(next_day).subtract(days=2).to_rfc3339_string()})
+        oauth = DeclarativeOauth2Authenticator(
+            token_refresh_endpoint="{{ config['refresh_endpoint'] }}",
+            client_id="{{ config['client_id'] }}",
+            client_secret="{{ config['client_secret'] }}",
+            refresh_token="{{ config['refresh_token'] }}",
+            config=config,
+            scopes=["scope1", "scope2"],
+            token_expiry_date="{{ config['token_expiry_date'] }}",
+            token_expiry_date_format=token_expiry_date_format,
+            refresh_request_body={
+                "custom_field": "{{ config['custom_field'] }}",
+                "another_field": "{{ config['another_field'] }}",
+                "scopes": ["no_override"],
+            },
+            options={},
+        )
+
+        resp.status_code = 200
+        mocker.patch.object(resp, "json", return_value={"access_token": "access_token", "expires_in": expires_in_response})
+        mocker.patch.object(requests, "request", side_effect=mock_request, autospec=True)
+        token = oauth.get_access_token()
+        assert "access_token" == token
+        assert oauth.get_token_expiry_date() == pendulum.parse(next_day)
+
 
 def mock_request(method, url, data):
     if url == "refresh_end":

airbyte-cdk/python/unit_tests/sources/declarative/schema/source_test/schemas/sample_stream.json

@@ -0,0 +1,12 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": ["null", "object"],
+  "properties": {
+    "type": {
+      "$ref": "sample_shared_schema.json"
+    },
+    "id": {
+      "type": ["null", "string"]
+    }
+  }
+}

airbyte-cdk/python/unit_tests/sources/declarative/schema/source_test/schemas/shared/sample_shared_schema.json

@@ -0,0 +1,11 @@
+{
+  "type": ["null", "object"],
+  "properties": {
+    "id_internal": {
+      "type": ["null", "integer"]
+    },
+    "name": {
+      "type": ["null", "string"]
+    }
+  }
+}