Ensure that bytes/str values are decoded to utf-8

iopipe/compat.py

@@ -3,13 +3,15 @@
 
 PY3 = sys.version_info[0] == 3
 
-if PY3:
+if PY3:  # pragma: no cover
     from urllib.parse import urlparse
 
+    binary_types = bytes
     string_types = (str,)
     StringIO = io.StringIO
-else:
+else:  # pragma: no cover
     from urlparse import urlparse  # noqa
 
+    binary_types = str
     string_types = (basestring,)  # noqa
     StringIO = io.BytesIO

iopipe/contrib/trace/auto_http.py

@@ -12,6 +12,7 @@
     BotocoreSession = None
 
 from iopipe.compat import urlparse
+from .util import ensure_utf8
 
 if Session is not None:
     original_session_send = Session.send
@@ -67,7 +68,7 @@ def patch_session_send(context, http_filter):
         return
 
     def send(self, *args, **kwargs):
-        id = str(uuid.uuid4())
+        id = ensure_utf8(str(uuid.uuid4()))
         with context.iopipe.mark(id):
             response = original_session_send(self, *args, **kwargs)
         trace = context.iopipe.mark.measure(id)
@@ -139,36 +140,36 @@ def collect_metrics_for_response(http_response, context, trace, http_filter):
         request_headers = []
         if hasattr(http_response.request, "headers"):
             request_headers = [
-                {"key": k, "string": v}
+                {"key": ensure_utf8(k), "string": ensure_utf8(v)}
                 for k, v in http_response.request.headers.items()
                 if k.lower() in INCLUDE_HEADERS
             ]
 
         request = Request(
-            hash=getattr(parsed_url, "fragment", None),
+            hash=ensure_utf8(getattr(parsed_url, "fragment", None)),
             headers=request_headers,
-            hostname=getattr(parsed_url, "hostname", None),
-            method=getattr(http_response.request, "method", None),
-            path=getattr(parsed_url, "path", None),
+            hostname=ensure_utf8(getattr(parsed_url, "hostname", None)),
+            method=ensure_utf8(getattr(http_response.request, "method", None)),
+            path=ensure_utf8(getattr(parsed_url, "path", None)),
             # TODO: Determine if this is redundant
-            pathname=getattr(parsed_url, "path", None),
-            port=getattr(parsed_url, "port", None),
-            protocol=getattr(parsed_url, "scheme", None),
-            query=getattr(parsed_url, "query", None),
-            url=getattr(http_response.request, "url", None),
+            pathname=ensure_utf8(getattr(parsed_url, "path", None)),
+            port=ensure_utf8(getattr(parsed_url, "port", None)),
+            protocol=ensure_utf8(getattr(parsed_url, "scheme", None)),
+            query=ensure_utf8(getattr(parsed_url, "query", None)),
+            url=ensure_utf8(getattr(http_response.request, "url", None)),
         )
 
     response_headers = []
     if hasattr(http_response, "headers"):
         response_headers = [
-            {"key": k, "string": v}
+            {"key": ensure_utf8(k), "string": ensure_utf8(v)}
             for k, v in http_response.headers.items()
             if k.lower() in INCLUDE_HEADERS
         ]
 
     response = Response(
         headers=response_headers,
-        statusCode=getattr(http_response, "status_code", None),
+        statusCode=ensure_utf8(getattr(http_response, "status_code", None)),
         statusMessage=None,
     )
 

iopipe/contrib/trace/util.py

@@ -1,3 +1,6 @@
+from iopipe.compat import binary_types
+
+
 def add_timeline_measures(timeline):
     entries = timeline.get_entries_by_type("mark")
     measures = timeline.get_entries_by_type("measure")
@@ -14,3 +17,10 @@ def add_timeline_measures(timeline):
             measure_name = "measure:%s" % base_name
             if end_name in entry_names and measure_name not in measure_names:
                 timeline.measure(measure_name, name, end_name)
+
+
+def ensure_utf8(v):
+    """Ensures that a binary value is decoded into utf-8"""
+    if isinstance(v, binary_types):
+        return v.decode("utf-8", "ignore")
+    return v

tests/contrib/trace/test_util.py

@@ -0,0 +1,6 @@
+from iopipe.compat import string_types
+from iopipe.contrib.trace.util import ensure_utf8
+
+
+def test_ensure_utf8():
+    assert isinstance(ensure_utf8("foobar".encode("utf-8")), string_types)