bugfix: support bytes from internal readings [VIO-1238]

synse_server/cmd/read.py

@@ -1,5 +1,6 @@
 
 import asyncio
+import json
 import math
 import queue
 import threading
@@ -25,6 +26,7 @@ def reading_to_dict(reading: api.V3Reading) -> Dict[str, Any]:
         The reading converted to its dictionary representation, conforming
         to the V3 API read schema.
     """
+
     # The reading value is stored in a protobuf oneof block - we need to
     # figure out which field it is so we can extract it. If no field is set,
     # take the reading value to be None.
@@ -33,6 +35,25 @@ def reading_to_dict(reading: api.V3Reading) -> Dict[str, Any]:
     if field is not None:
         value = getattr(reading, field)
 
+        # If the value returned is bytes, perform some additional checks.
+        # Bytes are not JSON-serializable, so we must convert it. We will
+        # default to bytes being converted to a string, but also try to
+        # load the bytes as JSON, in the event that the payload contains
+        # valid JSON.
+        if isinstance(value, bytes):
+            value = value.decode("utf-8")
+            try:
+                # json.loads will take string values (e.g. "1", "0.25", "null")
+                # and convert them to corresponding python types (1, 0.25, None).
+                # we don't want to do this for every byte string that comes through,
+                # as we may be expecting a string value for some bytes responses.
+                # Only capture the JSON if it renders out to a dict or list.
+                tmp = json.loads(value)
+                if isinstance(tmp, (dict, list)):
+                    value = tmp
+            except Exception:  # noqa
+                pass
+
         # Ensure the value is not NaN, as NaN is not a part of the
         # JSON spec and could cause clients to error.
         try:

tests/unit/cmd/test_read.py

@@ -1,5 +1,7 @@
 """Unit tests for the ``synse_server.cmd.read`` module."""
 
+from typing import Any
+
 import asynctest
 import pytest
 from synse_grpc import api, client
@@ -746,3 +748,80 @@ def test_reading_to_dict_5_float_not_nan(state_reading, reading_value: str) -> N
         'unit': None,
         'context': {},
     }
+
+
+@pytest.mark.parametrize(
+    'raw_value,expected', [
+        (b"", ""),
+        (b"abc", "abc"),
+        (b"1", "1"),
+        (b"0.25", "0.25"),
+        (b"null", "null"),
+    ]
+)
+def test_reading_to_dict_byte_string(raw_value: bytes, expected: Any, const_uuid: str) -> None:
+    """Convert a reading to a dictionary. Here, the reading value is typed as bytes
+    which are not JSON serializable. The method should convert the bytes to a string.
+
+    Regression for: https://vaporio.atlassian.net/browse/VIO-1238
+    """
+
+    msg = api.V3Reading(
+        id=const_uuid,
+        timestamp='2019-04-22T13:30:00Z',
+        type="example",
+        deviceType="example",
+        deviceInfo="An Example Device",
+        bytes_value=raw_value,
+    )
+
+    actual = reading_to_dict(msg)
+    assert actual == {
+        'device': const_uuid,
+        'timestamp': '2019-04-22T13:30:00Z',
+        'type': 'example',
+        'device_type': 'example',
+        'device_info': 'An Example Device',
+        'value': expected,
+        'unit': None,
+        'context': {},
+    }
+
+
+@pytest.mark.parametrize(
+    'raw_value,expected', [
+        (b"[]", []),
+        (b"{}", {}),
+        (b'{"foo": "bar"}', {"foo": "bar"}),
+        (b'{"foo": 1}', {"foo": 1}),
+        (b'{"foo": [true, false]}', {"foo": [True, False]}),
+    ]
+)
+def test_reading_to_dict_byte_json(raw_value: bytes, expected: Any, const_uuid: str) -> None:
+    """Convert a reading to a dictionary. Here, the reading value is typed as bytes
+    which are not JSON serializable. The method should convert the bytes to a string
+    and then load it as a valid JSON payload.
+
+    Regression for: https://vaporio.atlassian.net/browse/VIO-1238
+    """
+
+    msg = api.V3Reading(
+        id=const_uuid,
+        timestamp='2019-04-22T13:30:00Z',
+        type="example",
+        deviceType="example",
+        deviceInfo="An Example Device",
+        bytes_value=raw_value,
+    )
+
+    actual = reading_to_dict(msg)
+    assert actual == {
+        'device': const_uuid,
+        'timestamp': '2019-04-22T13:30:00Z',
+        'type': 'example',
+        'device_type': 'example',
+        'device_info': 'An Example Device',
+        'value': expected,
+        'unit': None,
+        'context': {},
+    }

tests/unit/conftest.py

@@ -218,3 +218,10 @@ def synse_app():
 
     TestManager(app.app)
     yield app.app
+
+
+@pytest.fixture()
+def const_uuid():
+    """Return a constant UUID."""
+
+    return "cadc0872-9e73-4122-bfa2-377d176374e0"