diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 76bb44ebfb9b..d8f5aafa7100 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.14.0 +Low-code: Add token_expiry_date_format to OAuth Authenticator. Resolve ref schema + ## 0.13.3 Fixed `StopIteration` exception for empty streams while `check_availability` runs. diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py index 97869cc9010a..a38900c22e1a 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py @@ -30,6 +30,7 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut config (Mapping[str, Any]): The user-provided configuration as specified by the source's spec scopes (Optional[List[str]]): The scopes to request token_expiry_date (Optional[Union[InterpolatedString, str]]): The access token expiration date + token_expiry_date_format str: format of the datetime; provide it if expires_in is returned in datetime instead of seconds refresh_request_body (Optional[Mapping[str, Any]]): The request body to send in the refresh request grant_type: The grant_type to request for access_token """ @@ -43,6 +44,7 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut scopes: Optional[List[str]] = None token_expiry_date: Optional[Union[InterpolatedString, str]] = None _token_expiry_date: pendulum.DateTime = field(init=False, repr=False, default=None) + token_expiry_date_format: str = None access_token_name: Union[InterpolatedString, str] = "access_token" expires_in_name: Union[InterpolatedString, str] = "expires_in" refresh_request_body: Optional[Mapping[str, Any]] = None @@ -94,8 +96,11 @@ def get_refresh_request_body(self) -> Mapping[str, Any]: def get_token_expiry_date(self) -> pendulum.DateTime: return self._token_expiry_date - def set_token_expiry_date(self, value: pendulum.DateTime): - self._token_expiry_date = value + def set_token_expiry_date(self, initial_time: pendulum.DateTime, value: Union[str, int]): + if self.token_expiry_date_format: + self._token_expiry_date = pendulum.from_format(value, self.token_expiry_date_format) + else: + self._token_expiry_date = initial_time.add(seconds=value) @property def access_token(self) -> str: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py index fa4e5a99a005..ba4fd30635ff 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py @@ -11,6 +11,7 @@ from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader from airbyte_cdk.sources.declarative.types import Config +from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader from dataclasses_jsonschema import JsonSchemaMixin @@ -30,7 +31,7 @@ def _default_file_path() -> str: @dataclass -class JsonFileSchemaLoader(SchemaLoader, JsonSchemaMixin): +class JsonFileSchemaLoader(ResourceSchemaLoader, SchemaLoader, JsonSchemaMixin): """ Loads the schema from a json file @@ -63,7 +64,8 @@ def get_json_schema(self) -> Mapping[str, Any]: raw_schema = json.loads(raw_json_file) except ValueError as err: raise RuntimeError(f"Invalid JSON file format for file {json_schema_path}") from err - return raw_schema + self.package_name = resource + return self._resolve_schema_references(raw_schema) def _get_json_filepath(self): return self.file_path.eval(self.config) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py index fab826bd7b46..65eb2bc91ca5 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py @@ -3,7 +3,7 @@ # from abc import abstractmethod -from typing import Any, List, Mapping, MutableMapping, Tuple +from typing import Any, List, Mapping, MutableMapping, Tuple, Union import pendulum import requests @@ -29,10 +29,10 @@ def get_auth_header(self) -> Mapping[str, Any]: def get_access_token(self) -> str: """Returns the access token""" if self.token_has_expired(): - t0 = pendulum.now() + current_datetime = pendulum.now() token, expires_in = self.refresh_access_token() self.access_token = token - self.set_token_expiry_date(t0.add(seconds=expires_in)) + self.set_token_expiry_date(current_datetime, expires_in) return self.access_token @@ -102,11 +102,11 @@ def get_scopes(self) -> List[str]: """List of requested scopes""" @abstractmethod - def get_token_expiry_date(self) -> pendulum.datetime: + def get_token_expiry_date(self) -> pendulum.DateTime: """Expiration date of the access token""" @abstractmethod - def set_token_expiry_date(self, value: pendulum.datetime): + def set_token_expiry_date(self, initial_time: pendulum.DateTime, value: Union[str, int]): """Setter for access token expiration date""" @abstractmethod diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py index 5be3b3d05d16..ca4398ddd08f 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py @@ -2,7 +2,7 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from typing import Any, List, Mapping, Sequence, Tuple +from typing import Any, List, Mapping, Sequence, Tuple, Union import dpath import pendulum @@ -25,6 +25,7 @@ def __init__( refresh_token: str, scopes: List[str] = None, token_expiry_date: pendulum.DateTime = None, + token_expiry_date_format: str = None, access_token_name: str = "access_token", expires_in_name: str = "expires_in", refresh_request_body: Mapping[str, Any] = None, @@ -41,6 +42,7 @@ def __init__( self._grant_type = grant_type self._token_expiry_date = token_expiry_date or pendulum.now().subtract(days=1) + self._token_expiry_date_format = token_expiry_date_format self._access_token = None def get_token_refresh_endpoint(self) -> str: @@ -73,8 +75,11 @@ def get_grant_type(self) -> str: def get_token_expiry_date(self) -> pendulum.DateTime: return self._token_expiry_date - def set_token_expiry_date(self, value: pendulum.DateTime): - self._token_expiry_date = value + def set_token_expiry_date(self, initial_time: pendulum.DateTime, value: Union[str, int]): + if self._token_expiry_date_format: + self._token_expiry_date = pendulum.from_format(value, self._token_expiry_date_format) + else: + self._token_expiry_date = initial_time.add(seconds=value) @property def access_token(self) -> str: @@ -100,6 +105,7 @@ def __init__( token_refresh_endpoint: str, scopes: List[str] = None, token_expiry_date: pendulum.DateTime = None, + token_expiry_date_format: str = None, access_token_name: str = "access_token", expires_in_name: str = "expires_in", refresh_token_name: str = "refresh_token", @@ -138,6 +144,7 @@ def __init__( self.get_refresh_token(), scopes, token_expiry_date, + token_expiry_date_format, access_token_name, expires_in_name, refresh_request_body, @@ -192,7 +199,7 @@ def get_access_token(self) -> str: t0 = pendulum.now() new_access_token, access_token_expires_in, new_refresh_token = self.refresh_access_token() self.access_token = new_access_token - self.set_token_expiry_date(t0.add(seconds=access_token_expires_in)) + self.set_token_expiry_date(t0, access_token_expires_in) self.set_refresh_token(new_refresh_token) return self.access_token diff --git a/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py b/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py index aea02ecec950..368a45225621 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py @@ -129,9 +129,9 @@ def get_schema(self, name: str) -> dict: except ValueError as err: raise RuntimeError(f"Invalid JSON file format for file {schema_filename}") from err - return self.__resolve_schema_references(raw_schema) + return self._resolve_schema_references(raw_schema) - def __resolve_schema_references(self, raw_schema: dict) -> dict: + def _resolve_schema_references(self, raw_schema: dict) -> dict: """ Resolve links to external references and move it to local "definitions" map. diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index ef72c6ae3735..b030f7b3654e 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.13.3", + version="0.14.0", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", @@ -63,6 +63,7 @@ python_requires=">=3.9", extras_require={ "dev": [ + "freezegun", "MyPy~=0.812", "pytest", "pytest-cov", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_oauth.py b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_oauth.py index fe3ea518611d..8ee9f3ad666a 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_oauth.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_oauth.py @@ -4,7 +4,9 @@ import logging +import freezegun import pendulum +import pytest import requests from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator from requests import Response @@ -90,6 +92,44 @@ def test_refresh_access_token(self, mocker): assert ("access_token", 1000) == token + @pytest.mark.parametrize( + "expires_in_response, token_expiry_date_format", + [ + (86400, None), + ("2020-01-02T00:00:00Z", "YYYY-MM-DDTHH:mm:ss[Z]"), + ('2020-01-02T00:00:00.000000+00:00', "YYYY-MM-DDTHH:mm:ss.SSSSSSZ"), + ("2020-01-02", "YYYY-MM-DD"), + ], + ids=["time_in_seconds", "rfc3339", "iso8601", "simple_date"] + ) + @freezegun.freeze_time("2020-01-01") + def test_refresh_access_token_expire_format(self, mocker, expires_in_response, token_expiry_date_format): + next_day = "2020-01-02T00:00:00Z" + config.update({"token_expiry_date": pendulum.parse(next_day).subtract(days=2).to_rfc3339_string()}) + oauth = DeclarativeOauth2Authenticator( + token_refresh_endpoint="{{ config['refresh_endpoint'] }}", + client_id="{{ config['client_id'] }}", + client_secret="{{ config['client_secret'] }}", + refresh_token="{{ config['refresh_token'] }}", + config=config, + scopes=["scope1", "scope2"], + token_expiry_date="{{ config['token_expiry_date'] }}", + token_expiry_date_format=token_expiry_date_format, + refresh_request_body={ + "custom_field": "{{ config['custom_field'] }}", + "another_field": "{{ config['another_field'] }}", + "scopes": ["no_override"], + }, + options={}, + ) + + resp.status_code = 200 + mocker.patch.object(resp, "json", return_value={"access_token": "access_token", "expires_in": expires_in_response}) + mocker.patch.object(requests, "request", side_effect=mock_request, autospec=True) + token = oauth.get_access_token() + assert "access_token" == token + assert oauth.get_token_expiry_date() == pendulum.parse(next_day) + def mock_request(method, url, data): if url == "refresh_end": diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/schema/source_test/schemas/sample_stream.json b/airbyte-cdk/python/unit_tests/sources/declarative/schema/source_test/schemas/sample_stream.json new file mode 100644 index 000000000000..6ef7fbed5577 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/schema/source_test/schemas/sample_stream.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "properties": { + "type": { + "$ref": "sample_shared_schema.json" + }, + "id": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/schema/source_test/schemas/shared/sample_shared_schema.json b/airbyte-cdk/python/unit_tests/sources/declarative/schema/source_test/schemas/shared/sample_shared_schema.json new file mode 100644 index 000000000000..95ea8a1655f2 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/schema/source_test/schemas/shared/sample_shared_schema.json @@ -0,0 +1,11 @@ +{ + "type": ["null", "object"], + "properties": { + "id_internal": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } +}