From 725b7b026194a7d08f056c1977985e8c14a22366 Mon Sep 17 00:00:00 2001 From: Curtis Mason Date: Thu, 13 Aug 2020 10:34:44 -0700 Subject: [PATCH 1/2] added exceptions and more indepth can_read Signed-off-by: Curtis Mason --- cloudevents/exceptions.py | 19 +++++++++++ cloudevents/http/event.py | 5 +-- cloudevents/http/http_methods.py | 11 ++++-- cloudevents/sdk/converters/binary.py | 10 +++--- cloudevents/sdk/converters/structured.py | 10 +++--- cloudevents/sdk/converters/util.py | 10 ++++++ cloudevents/sdk/event/base.py | 31 ++++++++++++----- cloudevents/tests/test_http_events.py | 43 +++++++++--------------- 8 files changed, 89 insertions(+), 50 deletions(-) create mode 100644 cloudevents/exceptions.py create mode 100644 cloudevents/sdk/converters/util.py diff --git a/cloudevents/exceptions.py b/cloudevents/exceptions.py new file mode 100644 index 00000000..5d2e191e --- /dev/null +++ b/cloudevents/exceptions.py @@ -0,0 +1,19 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +class CloudEventMissingRequiredFields(Exception): + pass + + +class CloudEventTypeErrorRequiredFields(Exception): + pass diff --git a/cloudevents/http/event.py b/cloudevents/http/event.py index a31b894b..77702c03 100644 --- a/cloudevents/http/event.py +++ b/cloudevents/http/event.py @@ -16,6 +16,7 @@ import typing import uuid +import cloudevents.exceptions as cloud_exceptions from cloudevents.http.mappings import _required_by_version @@ -57,14 +58,14 @@ def __init__( ).isoformat() if self._attributes["specversion"] not in _required_by_version: - raise ValueError( + raise cloud_exceptions.CloudEventMissingRequiredFields( f"Invalid specversion: {self._attributes['specversion']}" ) # There is no good way to default 'source' and 'type', so this # checks for those (or any new required attributes). required_set = _required_by_version[self._attributes["specversion"]] if not required_set <= self._attributes.keys(): - raise ValueError( + raise cloud_exceptions.CloudEventMissingRequiredFields( f"Missing required keys: {required_set - attributes.keys()}" ) diff --git a/cloudevents/http/http_methods.py b/cloudevents/http/http_methods.py index 113e1969..8436d69d 100644 --- a/cloudevents/http/http_methods.py +++ b/cloudevents/http/http_methods.py @@ -1,6 +1,7 @@ import json import typing +import cloudevents.exceptions as cloud_exceptions from cloudevents.http.event import CloudEvent from cloudevents.http.mappings import _marshaller_by_format, _obj_by_version from cloudevents.http.util import _json_or_string @@ -34,12 +35,16 @@ def from_http( specversion = raw_ce.get("specversion", None) if specversion is None: - raise ValueError("could not find specversion in HTTP request") + raise cloud_exceptions.CloudEventMissingRequiredFields( + "could not find specversion in HTTP request" + ) event_handler = _obj_by_version.get(specversion, None) if event_handler is None: - raise ValueError(f"found invalid specversion {specversion}") + raise cloud_exceptions.CloudEventTypeErrorRequiredFields( + f"found invalid specversion {specversion}" + ) event = marshall.FromRequest( event_handler(), headers, data, data_unmarshaller=data_unmarshaller @@ -71,7 +76,7 @@ def _to_http( data_marshaller = _marshaller_by_format[format] if event._attributes["specversion"] not in _obj_by_version: - raise ValueError( + raise cloud_exceptions.CloudEventTypeErrorRequiredFields( f"Unsupported specversion: {event._attributes['specversion']}" ) diff --git a/cloudevents/sdk/converters/binary.py b/cloudevents/sdk/converters/binary.py index 46277727..e45b9471 100644 --- a/cloudevents/sdk/converters/binary.py +++ b/cloudevents/sdk/converters/binary.py @@ -16,7 +16,7 @@ from cloudevents.sdk import exceptions, types from cloudevents.sdk.converters import base -from cloudevents.sdk.converters.structured import JSONHTTPCloudEventConverter +from cloudevents.sdk.converters.util import has_binary_headers from cloudevents.sdk.event import base as event_base from cloudevents.sdk.event import v1, v03 @@ -28,13 +28,11 @@ class BinaryHTTPCloudEventConverter(base.Converter): def can_read( self, - content_type: str, + content_type: str = None, headers: typing.Dict[str, str] = {"ce-specversion": None}, ) -> bool: - return ("ce-specversion" in headers) and not ( - isinstance(content_type, str) - and content_type.startswith(JSONHTTPCloudEventConverter.MIME_TYPE) - ) + + return has_binary_headers(headers) def event_supported(self, event: object) -> bool: return type(event) in self.SUPPORTED_VERSIONS diff --git a/cloudevents/sdk/converters/structured.py b/cloudevents/sdk/converters/structured.py index d6ba6548..eba92797 100644 --- a/cloudevents/sdk/converters/structured.py +++ b/cloudevents/sdk/converters/structured.py @@ -16,23 +16,25 @@ from cloudevents.sdk import types from cloudevents.sdk.converters import base +from cloudevents.sdk.converters.binary import BinaryHTTPCloudEventConverter +from cloudevents.sdk.converters.util import has_binary_headers from cloudevents.sdk.event import base as event_base +# TODO: Singleton? class JSONHTTPCloudEventConverter(base.Converter): TYPE = "structured" MIME_TYPE = "application/cloudevents+json" def can_read( - self, - content_type: str, - headers: typing.Dict[str, str] = {"ce-specversion": None}, + self, content_type: str, headers: typing.Dict[str, str] = {}, ) -> bool: return ( isinstance(content_type, str) and content_type.startswith(self.MIME_TYPE) - ) or ("ce-specversion" not in headers) + or not has_binary_headers(headers) + ) def event_supported(self, event: object) -> bool: # structured format supported by both spec 0.1 and 0.2 diff --git a/cloudevents/sdk/converters/util.py b/cloudevents/sdk/converters/util.py new file mode 100644 index 00000000..b31c39c8 --- /dev/null +++ b/cloudevents/sdk/converters/util.py @@ -0,0 +1,10 @@ +import typing + + +def has_binary_headers(headers: typing.Dict[str, str]) -> bool: + return ( + "ce-specversion" in headers + and "ce-source" in headers + and "ce-type" in headers + and "ce-id" in headers + ) diff --git a/cloudevents/sdk/event/base.py b/cloudevents/sdk/event/base.py index 2004dbbe..ed5592aa 100644 --- a/cloudevents/sdk/event/base.py +++ b/cloudevents/sdk/event/base.py @@ -16,10 +16,12 @@ import json import typing +import cloudevents.exceptions as cloud_exceptions from cloudevents.sdk import types - # TODO(slinkydeveloper) is this really needed? + + class EventGetterSetter(object): # ce-specversion @@ -159,6 +161,9 @@ def content_type(self, value: str): class BaseEvent(EventGetterSetter): + _ce_required_fields = set() + _ce_optional_fields = set() + def Properties(self, with_nullable=False) -> dict: props = dict() for name, value in self.__dict__.items(): @@ -193,7 +198,10 @@ def Set(self, key: str, value: object): def MarshalJSON(self, data_marshaller: types.MarshallerType) -> str: if data_marshaller is None: - data_marshaller = lambda x: x # noqa: E731 + + def data_marshaller(x): + return x # noqa: E731 + props = self.Properties() if "data" in props: data = data_marshaller(props.pop("data")) @@ -215,7 +223,9 @@ def UnmarshalJSON( missing_fields = self._ce_required_fields - raw_ce.keys() if len(missing_fields) > 0: - raise ValueError(f"Missing required attributes: {missing_fields}") + raise cloud_exceptions.CloudEventMissingRequiredFields( + f"Missing required attributes: {missing_fields}" + ) for name, value in raw_ce.items(): if name == "data": @@ -233,8 +243,16 @@ def UnmarshalBinary( body: typing.Union[bytes, str], data_unmarshaller: types.UnmarshallerType, ): - if "ce-specversion" not in headers: - raise ValueError("Missing required attribute: 'specversion'") + required_binary_fields = { + f"ce-{field}" for field in self._ce_required_fields + } + missing_fields = required_binary_fields - headers.keys() + + if len(missing_fields) > 0: + raise cloud_exceptions.CloudEventMissingRequiredFields( + f"Missing required attributes: {missing_fields}" + ) + for header, value in headers.items(): header = header.lower() if header == "content-type": @@ -242,9 +260,6 @@ def UnmarshalBinary( elif header.startswith("ce-"): self.Set(header[3:], value) self.Set("data", data_unmarshaller(body)) - missing_attrs = self._ce_required_fields - self.Properties().keys() - if len(missing_attrs) > 0: - raise ValueError(f"Missing required attributes: {missing_attrs}") def MarshalBinary( self, data_marshaller: types.MarshallerType diff --git a/cloudevents/tests/test_http_events.py b/cloudevents/tests/test_http_events.py index 883e01b8..9bfd7d33 100644 --- a/cloudevents/tests/test_http_events.py +++ b/cloudevents/tests/test_http_events.py @@ -20,6 +20,7 @@ import pytest from sanic import Sanic, response +import cloudevents.exceptions as cloud_exceptions from cloudevents.http import ( CloudEvent, from_http, @@ -47,7 +48,7 @@ }, ] -invalid_cloudevent_request_bodie = [ +invalid_cloudevent_request_body = [ { "source": "", "type": "cloudevent.event.type", @@ -73,9 +74,13 @@ def post(url, headers, data): @app.route("/event", ["POST"]) async def echo(request): - decoder = None if "binary-payload" in request.headers: - decoder = lambda x: x + + def decoder(x): + return x + + else: + decoder = None event = from_http( request.body, headers=dict(request.headers), data_unmarshaller=decoder ) @@ -87,21 +92,22 @@ async def echo(request): return response.raw(data, headers={k: event[k] for k in event}) -@pytest.mark.parametrize("body", invalid_cloudevent_request_bodie) +@pytest.mark.parametrize("body", invalid_cloudevent_request_body) def test_missing_required_fields_structured(body): - with pytest.raises((TypeError, NotImplementedError)): + with pytest.raises(cloud_exceptions.CloudEventMissingRequiredFields): # CloudEvent constructor throws TypeError if missing required field # and NotImplementedError because structured calls aren't # implemented. In this instance one of the required keys should have # prefix e-id instead of ce-id therefore it should throw _ = from_http( - json.dumps(body), attributes={"Content-Type": "application/json"} + json.dumps(body), + headers={"Content-Type": "application/cloudevents+json"}, ) @pytest.mark.parametrize("headers", invalid_test_headers) def test_missing_required_fields_binary(headers): - with pytest.raises((ValueError)): + with pytest.raises(cloud_exceptions.CloudEventMissingRequiredFields): # CloudEvent constructor throws TypeError if missing required field # and NotImplementedError because structured calls aren't # implemented. In this instance one of the required keys should have @@ -165,7 +171,7 @@ def test_emit_structured_event(specversion): @pytest.mark.parametrize("specversion", ["1.0", "0.3"]) def test_roundtrip_non_json_event(converter, specversion): input_data = io.BytesIO() - for i in range(100): + for _ in range(100): for j in range(20): assert 1 == input_data.write(j.to_bytes(1, byteorder="big")) compressed_data = bz2.compress(input_data.getvalue()) @@ -201,7 +207,7 @@ def test_missing_ce_prefix_binary_event(specversion): # breaking prefix e.g. e-id instead of ce-id prefixed_headers[key[1:]] = headers[key] - with pytest.raises(ValueError): + with pytest.raises(cloud_exceptions.CloudEventMissingRequiredFields): # CloudEvent constructor throws TypeError if missing required field # and NotImplementedError because structured calls aren't # implemented. In this instance one of the required keys should have @@ -278,7 +284,7 @@ def test_empty_data_structured_event(specversion): # Testing if cloudevent breaks when no structured data field present attributes = { "specversion": specversion, - "datacontenttype": "application/json", + "datacontenttype": "application/cloudevents+json", "type": "word.found.name", "id": "96fb5f0b-001e-0108-6dfe-da6e2806f124", "time": "2018-10-23T12:28:22.4579346Z", @@ -308,7 +314,6 @@ def test_empty_data_binary_event(specversion): def test_valid_structured_events(specversion): # Test creating multiple cloud events events_queue = [] - headers = {} num_cloudevents = 30 for i in range(num_cloudevents): event = { @@ -335,9 +340,6 @@ def test_valid_structured_events(specversion): @pytest.mark.parametrize("specversion", ["1.0", "0.3"]) def test_structured_no_content_type(specversion): # Test creating multiple cloud events - events_queue = [] - headers = {} - num_cloudevents = 30 data = { "id": "id", "source": "source.com.test", @@ -373,19 +375,6 @@ def test_is_binary(): assert not converters.is_binary(headers) -def test_is_structured(): - headers = { - "Content-Type": "application/cloudevents+json", - } - assert converters.is_structured(headers) - - headers = {} - assert converters.is_structured(headers) - - headers = {"ce-specversion": "1.0"} - assert not converters.is_structured(headers) - - @pytest.mark.parametrize("specversion", ["1.0", "0.3"]) def test_cloudevent_repr(specversion): headers = { From 2790cae90033d07e33073d3ad0fee398e83c23f3 Mon Sep 17 00:00:00 2001 From: Curtis Mason Date: Thu, 13 Aug 2020 10:44:38 -0700 Subject: [PATCH 2/2] moved is_binary, is_structured into http module Signed-off-by: Curtis Mason --- cloudevents/http/__init__.py | 1 + cloudevents/http/event_type.py | 29 ++++++++++++++++++++++++++ cloudevents/http/http_methods.py | 3 ++- cloudevents/sdk/converters/__init__.py | 29 -------------------------- cloudevents/tests/test_http_events.py | 7 ++++--- 5 files changed, 36 insertions(+), 33 deletions(-) create mode 100644 cloudevents/http/event_type.py diff --git a/cloudevents/http/__init__.py b/cloudevents/http/__init__.py index 80fc5a74..d7c62ec4 100644 --- a/cloudevents/http/__init__.py +++ b/cloudevents/http/__init__.py @@ -15,6 +15,7 @@ import typing from cloudevents.http.event import CloudEvent +from cloudevents.http.event_type import is_binary, is_structured from cloudevents.http.http_methods import ( from_http, to_binary_http, diff --git a/cloudevents/http/event_type.py b/cloudevents/http/event_type.py new file mode 100644 index 00000000..fe6c0268 --- /dev/null +++ b/cloudevents/http/event_type.py @@ -0,0 +1,29 @@ +import typing + +from cloudevents.sdk.converters import binary, structured + + +def is_binary(headers: typing.Dict[str, str]) -> bool: + """Uses internal marshallers to determine whether this event is binary + :param headers: the HTTP headers + :type headers: typing.Dict[str, str] + :returns bool: returns a bool indicating whether the headers indicate a binary event type + """ + headers = {key.lower(): value for key, value in headers.items()} + content_type = headers.get("content-type", "") + binary_parser = binary.BinaryHTTPCloudEventConverter() + return binary_parser.can_read(content_type=content_type, headers=headers) + + +def is_structured(headers: typing.Dict[str, str]) -> bool: + """Uses internal marshallers to determine whether this event is structured + :param headers: the HTTP headers + :type headers: typing.Dict[str, str] + :returns bool: returns a bool indicating whether the headers indicate a structured event type + """ + headers = {key.lower(): value for key, value in headers.items()} + content_type = headers.get("content-type", "") + structured_parser = structured.JSONHTTPCloudEventConverter() + return structured_parser.can_read( + content_type=content_type, headers=headers + ) diff --git a/cloudevents/http/http_methods.py b/cloudevents/http/http_methods.py index 8436d69d..6f7b68d9 100644 --- a/cloudevents/http/http_methods.py +++ b/cloudevents/http/http_methods.py @@ -3,6 +3,7 @@ import cloudevents.exceptions as cloud_exceptions from cloudevents.http.event import CloudEvent +from cloudevents.http.event_type import is_binary, is_structured from cloudevents.http.mappings import _marshaller_by_format, _obj_by_version from cloudevents.http.util import _json_or_string from cloudevents.sdk import converters, marshaller, types @@ -28,7 +29,7 @@ def from_http( marshall = marshaller.NewDefaultHTTPMarshaller() - if converters.is_binary(headers): + if is_binary(headers): specversion = headers.get("ce-specversion", None) else: raw_ce = json.loads(data) diff --git a/cloudevents/sdk/converters/__init__.py b/cloudevents/sdk/converters/__init__.py index 289cfab4..936e8084 100644 --- a/cloudevents/sdk/converters/__init__.py +++ b/cloudevents/sdk/converters/__init__.py @@ -11,36 +11,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - -import typing - from cloudevents.sdk.converters import binary, structured TypeBinary = binary.BinaryHTTPCloudEventConverter.TYPE TypeStructured = structured.JSONHTTPCloudEventConverter.TYPE - - -def is_binary(headers: typing.Dict[str, str]) -> bool: - """Uses internal marshallers to determine whether this event is binary - :param headers: the HTTP headers - :type headers: typing.Dict[str, str] - :returns bool: returns a bool indicating whether the headers indicate a binary event type - """ - headers = {key.lower(): value for key, value in headers.items()} - content_type = headers.get("content-type", "") - binary_parser = binary.BinaryHTTPCloudEventConverter() - return binary_parser.can_read(content_type=content_type, headers=headers) - - -def is_structured(headers: typing.Dict[str, str]) -> bool: - """Uses internal marshallers to determine whether this event is structured - :param headers: the HTTP headers - :type headers: typing.Dict[str, str] - :returns bool: returns a bool indicating whether the headers indicate a structured event type - """ - headers = {key.lower(): value for key, value in headers.items()} - content_type = headers.get("content-type", "") - structured_parser = structured.JSONHTTPCloudEventConverter() - return structured_parser.can_read( - content_type=content_type, headers=headers - ) diff --git a/cloudevents/tests/test_http_events.py b/cloudevents/tests/test_http_events.py index 9bfd7d33..6465a1ac 100644 --- a/cloudevents/tests/test_http_events.py +++ b/cloudevents/tests/test_http_events.py @@ -24,6 +24,7 @@ from cloudevents.http import ( CloudEvent, from_http, + is_binary, to_binary_http, to_structured_http, ) @@ -364,15 +365,15 @@ def test_is_binary(): "ce-specversion": "1.0", "Content-Type": "text/plain", } - assert converters.is_binary(headers) + assert is_binary(headers) headers = { "Content-Type": "application/cloudevents+json", } - assert not converters.is_binary(headers) + assert not is_binary(headers) headers = {} - assert not converters.is_binary(headers) + assert not is_binary(headers) @pytest.mark.parametrize("specversion", ["1.0", "0.3"])