diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 67cb1b65ad8a..339c2cff8835 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1510,7 +1510,7 @@ - name: S3 sourceDefinitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2 dockerRepository: airbyte/source-s3 - dockerImageTag: 0.1.29 + dockerImageTag: 0.1.30 documentationUrl: https://docs.airbyte.com/integrations/sources/s3 icon: s3.svg sourceType: file @@ -1558,7 +1558,7 @@ - name: Sendgrid sourceDefinitionId: fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87 dockerRepository: airbyte/source-sendgrid - dockerImageTag: 0.2.16 + dockerImageTag: 0.3.0 documentationUrl: https://docs.airbyte.com/integrations/sources/sendgrid icon: sendgrid.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index d9ac4a58f00e..ad97f186b9ca 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -12660,7 +12660,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-s3:0.1.29" +- dockerImage: "airbyte/source-s3:0.1.30" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/s3" changelogUrl: "https://docs.airbyte.com/integrations/sources/s3" @@ -12869,14 +12869,10 @@ - "infer" - "error" order: 1 - allOf: - - title: "UnexpectedFieldBehaviorEnum" - description: "An enumeration." - enum: - - "ignore" - - "infer" - - "error" - type: "string" + enum: + - "ignore" + - "infer" + - "error" block_size: title: "Block Size" description: "The chunk size in bytes to process at a time in memory\ @@ -13236,7 +13232,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-sendgrid:0.2.16" +- dockerImage: "airbyte/source-sendgrid:0.3.0" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/sendgrid" connectionSpecification: @@ -13256,9 +13252,7 @@ order: 0 start_time: title: "Start time" - type: - - "integer" - - "string" + type: "string" description: "Start time in ISO8601 format. Any data before this time point\ \ will not be replicated." examples: diff --git a/airbyte-integrations/connectors/source-s3/Dockerfile b/airbyte-integrations/connectors/source-s3/Dockerfile index 718d25530ed4..0e03ad049bbc 100644 --- a/airbyte-integrations/connectors/source-s3/Dockerfile +++ b/airbyte-integrations/connectors/source-s3/Dockerfile @@ -17,5 +17,5 @@ COPY source_s3 ./source_s3 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.29 +LABEL io.airbyte.version=0.1.30 LABEL io.airbyte.name=airbyte/source-s3 diff --git a/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml b/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml index 825587b33a7a..95de536ab6f5 100644 --- a/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml @@ -4,7 +4,8 @@ acceptance_tests: spec: tests: - backward_compatibility_tests_config: - disable_for_version: 0.1.26 + # skip because allOf gets converted into enum + disable_for_version: 0.1.29 spec_path: integration_tests/spec.json connection: diff --git a/airbyte-integrations/connectors/source-s3/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-s3/acceptance-test-docker.sh old mode 100644 new mode 100755 diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/spec.json b/airbyte-integrations/connectors/source-s3/integration_tests/spec.json index 8aa6266bd205..c82b2d38c04a 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-s3/integration_tests/spec.json @@ -186,14 +186,7 @@ "default": "infer", "examples": ["ignore", "infer", "error"], "order": 1, - "allOf": [ - { - "title": "UnexpectedFieldBehaviorEnum", - "description": "An enumeration.", - "enum": ["ignore", "infer", "error"], - "type": "string" - } - ] + "enum": ["ignore", "infer", "error"] }, "block_size": { "title": "Block Size", diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/spec.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/spec.py index c1ffab7077cd..680fcbb6fec7 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/spec.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/spec.py @@ -87,6 +87,21 @@ def change_format_to_oneOf(schema: dict) -> dict: schema["properties"][prop]["oneOf"] = schema["properties"][prop].pop("anyOf") return schema + @staticmethod + def remove_enum_allOf(schema: dict) -> dict: + """ + allOfs are not supported by the UI, but pydantic is automatically writing them for enums. + Unpack them into the root + """ + objects_to_check = schema["properties"]["format"]["oneOf"] + for object in objects_to_check: + for key in object["properties"]: + property = object["properties"][key] + if "allOf" in property and "enum" in property["allOf"][0]: + property["enum"] = property["allOf"][0]["enum"] + property.pop("allOf") + return schema + @staticmethod def check_provider_added(schema: dict) -> None: if "provider" not in schema["properties"]: @@ -110,4 +125,5 @@ def schema(cls, *args: Any, **kwargs: Any) -> Dict[str, Any]: cls.check_provider_added(schema) schema = cls.change_format_to_oneOf(schema) schema = cls.resolve_refs(schema) + schema = cls.remove_enum_allOf(schema) return schema diff --git a/airbyte-integrations/connectors/source-sendgrid/Dockerfile b/airbyte-integrations/connectors/source-sendgrid/Dockerfile index 9fb8c810bb2f..e109ca825f66 100644 --- a/airbyte-integrations/connectors/source-sendgrid/Dockerfile +++ b/airbyte-integrations/connectors/source-sendgrid/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.16 +LABEL io.airbyte.version=0.3.0 LABEL io.airbyte.name=airbyte/source-sendgrid diff --git a/airbyte-integrations/connectors/source-sendgrid/acceptance-test-config.yml b/airbyte-integrations/connectors/source-sendgrid/acceptance-test-config.yml index f2fe28771e20..6b81ae9eb11c 100644 --- a/airbyte-integrations/connectors/source-sendgrid/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-sendgrid/acceptance-test-config.yml @@ -2,6 +2,8 @@ connector_image: airbyte/source-sendgrid:dev tests: spec: - spec_path: "source_sendgrid/spec.json" + backward_compatibility_tests_config: + disable_for_version: "0.2.16" connection: - config_path: "secrets/config.json" status: "succeed" diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json index 214e5045dc40..cf97393ced64 100644 --- a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json +++ b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json @@ -16,7 +16,7 @@ }, "start_time": { "title": "Start time", - "type": ["integer", "string"], + "type": "string", "description": "Start time in ISO8601 format. Any data before this time point will not be replicated.", "examples": [ "2021-12-12", diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py index fef1060c8ca5..74dfc9ab8f2a 100644 --- a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py +++ b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py @@ -5,7 +5,7 @@ import datetime import urllib from abc import ABC, abstractmethod -from typing import Any, Iterable, Mapping, MutableMapping, Optional, Union +from typing import Any, Iterable, Mapping, MutableMapping, Optional import pendulum import requests @@ -91,7 +91,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, class SendgridStreamIncrementalMixin(HttpStream, ABC): cursor_field = "created" - def __init__(self, start_time: Optional[Union[int, str]], **kwargs): + def __init__(self, start_time: Optional[str], **kwargs): super().__init__(**kwargs) self._start_time = start_time or 0 if isinstance(self._start_time, str): diff --git a/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py index cf8af51bfeb8..d322289b7a81 100644 --- a/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py @@ -28,6 +28,7 @@ ) FAKE_NOW = pendulum.DateTime(2022, 1, 1, tzinfo=pendulum.timezone("utc")) +FAKE_NOW_ISO_STRING = FAKE_NOW.to_iso8601_string() @pytest.fixture(name="sendgrid_stream") @@ -59,7 +60,7 @@ def test_source_wrong_credentials(): def test_messages_stream_request_params(mock_pendulum_now): - start_time = 1558359830 + start_time = "2019-05-20T13:43:50.000Z" stream = Messages(start_time) state = {"last_event_time": 1558359000} request_params = stream.request_params(state) @@ -70,7 +71,7 @@ def test_messages_stream_request_params(mock_pendulum_now): def test_streams(): - streams = SourceSendgrid().streams(config={"apikey": "wrong.api.key123", "start_time": FAKE_NOW}) + streams = SourceSendgrid().streams(config={"apikey": "wrong.api.key123", "start_time": FAKE_NOW_ISO_STRING}) assert len(streams) == 15 @@ -89,10 +90,10 @@ def test_pagination(mocker): @patch.multiple(SendgridStreamIncrementalMixin, __abstractmethods__=set()) def test_stream_state(): - stream = SendgridStreamIncrementalMixin(start_time=FAKE_NOW) + stream = SendgridStreamIncrementalMixin(start_time=FAKE_NOW_ISO_STRING) state = {} request_params = stream.request_params(stream_state=state) - assert request_params == {"end_time": pendulum.now().int_timestamp, "start_time": FAKE_NOW} + assert request_params == {"end_time": pendulum.now().int_timestamp, "start_time": int(FAKE_NOW.timestamp())} @pytest.mark.parametrize( diff --git a/docs/integrations/sources/s3.md b/docs/integrations/sources/s3.md index b24070c84c75..cdd3333b720c 100644 --- a/docs/integrations/sources/s3.md +++ b/docs/integrations/sources/s3.md @@ -209,6 +209,7 @@ The Jsonl parser uses pyarrow hence,only the line-delimited JSON format is suppo | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------------| +| 0.1.30 | 2023-01-25 | [21587](https://github.com/airbytehq/airbyte/pull/21587) | Make sure spec works as expected in UI | | 0.1.29 | 2023-01-19 | [21604](https://github.com/airbytehq/airbyte/pull/21604) | Handle OSError: skip unreachable keys and keep working on accessible ones. Warn a customer | | 0.1.28 | 2023-01-10 | [21210](https://github.com/airbytehq/airbyte/pull/21210) | Update block size for json file format | | 0.1.27 | 2022-12-08 | [20262](https://github.com/airbytehq/airbyte/pull/20262) | Check config settings for CSV file format | diff --git a/docs/integrations/sources/sendgrid.md b/docs/integrations/sources/sendgrid.md index 1d1fdf90b072..6e6c2b5c93a0 100644 --- a/docs/integrations/sources/sendgrid.md +++ b/docs/integrations/sources/sendgrid.md @@ -78,6 +78,7 @@ The connector is restricted by normal Sendgrid [requests limitation](https://sen | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------| +| 0.3.0 | 2023-01-25 | [21587](https://github.com/airbytehq/airbyte/pull/21587) | Make sure spec works as expected in UI - make starttime parameter an ISO string instead of a timestamp | | 0.2.16 | 2022-11-02 | [18847](https://github.com/airbytehq/airbyte/pull/18847) | Skip the stream on `400, 401 - authorization required` with log message | | 0.2.15 | 2022-10-19 | [18182](https://github.com/airbytehq/airbyte/pull/18182) | Mark the sendgrid api key secret in the spec | | 0.2.14 | 2022-09-07 | [16400](https://github.com/airbytehq/airbyte/pull/16400) | Change Start Time config parameter to datetime string |