Implement draft 10 of the http2 spec

.coveragerc

@@ -2,3 +2,4 @@
 omit =
     hyper/compat.py
     hyper/httplib_compat.py
+    hyper/ssl_compat.py

.gitignore

@@ -1,3 +1,4 @@
+build/
 env/
 dist/
 *.egg-info/

hyper/compat.py

@@ -5,19 +5,30 @@
 
 Normalizes the Python 2/3 API for internal use.
 """
+from contextlib import contextmanager
 import sys
 import zlib
 
-# Syntax sugar.
-_ver = sys.version_info
+try:
+    from . import ssl_compat
+except ImportError:
+    # TODO log?
+    ssl_compat = None
 
-#: Python 2.x?
-is_py2 = (_ver[0] == 2)
+_ver = sys.version_info
+is_py2 = _ver[0] == 2
+is_py3 = _ver[0] == 3
+is_py3_3 = is_py3 and _ver[1] == 3
 
-#: Python 3.x?
-is_py3 = (_ver[0] == 3)
+@contextmanager
+def ignore_missing():
+    try:
+        yield
+    except (AttributeError, NotImplementedError):  # pragma: no cover
+        pass
 
 if is_py2:
+    ssl = ssl_compat
     from urlparse import urlparse
 
     def to_byte(char):
@@ -41,3 +52,8 @@ def decode_hex(b):
         return bytes.fromhex(b)
 
     zlib_compressobj = zlib.compressobj
+
+    if is_py3_3:
+        ssl = ssl_compat
+    else:
+        import ssl

hyper/http20/frame.py

@@ -12,7 +12,6 @@
 # The maximum length of a frame. Some frames have shorter maximum lengths.
 FRAME_MAX_LEN = (2 ** 14) - 1
 
-
 class Frame(object):
     """
     The base class for all HTTP/2.0 frames.
@@ -82,26 +81,49 @@ class DataFrame(Frame):
     associated with a stream. One or more DATA frames are used, for instance,
     to carry HTTP request or response payloads.
     """
-    defined_flags = [('END_STREAM', 0x01)]
+    defined_flags = [('END_STREAM', 0x01), ('END_SEGMENT', 0x02),
+                     ('PAD_LOW', 0x10), ('PAD_HIGH', 0x20)]
 
     type = 0
 
     def __init__(self, stream_id):
         super(DataFrame, self).__init__(stream_id)
 
         self.data = b''
+        self.low_padding = 0
+        self.high_padding = 0
 
         # Data frames may not be stream 0.
         if not self.stream_id:
             raise ValueError()
 
     def serialize(self):
-        data = self.build_frame_header(len(self.data))
-        data += self.data
-        return data
+        padding_data = b''
+        if 'PAD_LOW' in self.flags:
+            if 'PAD_HIGH' in self.flags:
+                padding_data = struct.pack('!BB', self.high_padding, self.low_padding)
+            else:
+                padding_data = struct.pack('!B', self.low_padding)
+        padding = b'\0' * self.total_padding
+        body = b''.join([padding_data, self.data, padding])
+        header = self.build_frame_header(len(body))
+        return header + body
 
     def parse_body(self, data):
-        self.data = data
+        padding_data_length = 0
+        if 'PAD_LOW' in self.flags:
+            if 'PAD_HIGH' in self.flags:
+                self.high_padding, self.low_padding = struct.unpack('!BB', data[:2])
+                padding_data_length = 2
+            else:
+                self.low_padding = struct.unpack('!B', data[:1])[0]
+                padding_data_length = 1
+        self.data = data[padding_data_length:len(data)-self.total_padding]
+
+    @property
+    def total_padding(self):
+        """Return the total length of the padding, if any."""
+        return (self.high_padding << 8) + self.low_padding
 
 
 class PriorityFrame(Frame):
@@ -187,9 +209,8 @@ class SettingsFrame(Frame):
     # attributes.
     HEADER_TABLE_SIZE      = 0x01
     ENABLE_PUSH            = 0x02
-    MAX_CONCURRENT_STREAMS = 0x04
-    INITIAL_WINDOW_SIZE    = 0x07
-    FLOW_CONTROL_OPTIONS   = 0x0A
+    MAX_CONCURRENT_STREAMS = 0x03
+    INITIAL_WINDOW_SIZE    = 0x04
 
     def __init__(self, stream_id):
         super(SettingsFrame, self).__init__(stream_id)
@@ -201,19 +222,19 @@ def __init__(self, stream_id):
             raise ValueError()
 
     def serialize(self):
-        # Each setting consumes 8 bytes.
-        length = len(self.settings) * 8
+        # Each setting consumes 5 bytes.
+        length = len(self.settings) * 5
 
         data = self.build_frame_header(length)
 
         for setting, value in self.settings.items():
-            data += struct.pack("!LL", setting & 0x00FFFFFF, value)
+            data += struct.pack("!BL", setting & 0xFF, value)
 
         return data
 
     def parse_body(self, data):
-        for i in range(0, len(data), 8):
-            name, value = struct.unpack("!LL", data[i:i+8])
+        for i in range(0, len(data), 5):
+            name, value = struct.unpack("!BL", data[i:i+5])
             self.settings[name] = value
 
 
@@ -315,7 +336,7 @@ class WindowUpdateFrame(Frame):
     can indirectly cause the propagation of flow control information toward the
     original sender.
     """
-    type = 0x09
+    type = 0x08
 
     def __init__(self, stream_id):
         super(WindowUpdateFrame, self).__init__(stream_id)
@@ -348,8 +369,11 @@ class HeadersFrame(DataFrame):
 
     defined_flags = [
         ('END_STREAM', 0x01),
+        ('END_SEGMENT', 0x02),
         ('END_HEADERS', 0x04),
-        ('PRIORITY', 0x08)
+        ('PRIORITY', 0x08),
+        ('PAD_LOW', 0x10),
+        ('PAD_HIGH', 0x20),
     ]
 
     def __init__(self, stream_id):
@@ -386,21 +410,22 @@ class ContinuationFrame(DataFrame):
     Much like the HEADERS frame, hyper treats this as an opaque data frame with
     different flags and a different type.
     """
-    type = 0x0A
+    type = 0x09
 
-    defined_flags = [('END_HEADERS', 0x04)]
+    defined_flags = [('END_HEADERS', 0x04), ('PAD_LOW', 0x10), ('PAD_HIGH', 0x20)]
 
 
 # A map of type byte to frame class.
-FRAMES = {
-    0x00: DataFrame,
-    0x01: HeadersFrame,
-    0x02: PriorityFrame,
-    0x03: RstStreamFrame,
-    0x04: SettingsFrame,
-    0x05: PushPromiseFrame,
-    0x06: PingFrame,
-    0x07: GoAwayFrame,
-    0x09: WindowUpdateFrame,
-    0x0A: ContinuationFrame
-}
+_FRAME_CLASSES = [
+    DataFrame,
+    HeadersFrame,
+    PriorityFrame,
+    RstStreamFrame,
+    SettingsFrame,
+    PushPromiseFrame,
+    PingFrame,
+    GoAwayFrame,
+    WindowUpdateFrame,
+    ContinuationFrame,
+]
+FRAMES = {cls.type: cls for cls in _FRAME_CLASSES}

hyper/http20/hpack.py

@@ -637,7 +637,7 @@ def _decode_indexed(self, data):
         # set. Otherwise, decode it as an integer with a 7-bit prefix: that's
         # our new header table max size.
         if not index:
-            next_byte = data[consumed]
+            next_byte = to_byte(data[consumed])
 
             if next_byte & 0x80:
                 self.reference_set = set()

hyper/http20/stream.py

@@ -140,6 +140,11 @@ def listlen(list):
             # Append the data to the buffer.
             data.append(frame.data)
 
+            # Increase the window size. Only do this if the data frame contains
+            # actual data.
+            size = len(frame.data) + frame.total_padding
+            increment = self._in_window_manager._handle_frame(size)
+
             # If that was the last frame, we're done here.
             if 'END_STREAM' in frame.flags:
                 self.state = (
@@ -148,9 +153,6 @@ def listlen(list):
                 )
                 break
 
-            # Increase the window size. Only do this if the data frame contains
-            # actual data.
-            increment = self._in_window_manager._handle_frame(len(frame.data))
             if increment:
                 w = WindowUpdateFrame(self.stream_id)
                 w.window_increment = increment

hyper/http20/tls.py

@@ -5,75 +5,57 @@
 
 Contains the TLS/SSL logic for use in hyper.
 """
-import ssl
 import os.path as path
 
-from ..compat import is_py3
+from ..compat import ignore_missing, ssl
 
 
-# Right now we support draft 9.
-SUPPORTED_PROTOCOLS = ['http/1.1', 'HTTP-draft-09/2.0']
+NPN_PROTOCOL = 'h2-10'
+SUPPORTED_NPN_PROTOCOLS = ['http/1.1', NPN_PROTOCOL]
 
 
 # We have a singleton SSLContext object. There's no reason to be creating one
-# per connection. We're using v23 right now until someone gives me a reason not
-# to.
+# per connection.
 _context = None
 
-# Exposed here so it can be monkey-patched in integration tests.
-_verify_mode = ssl.CERT_REQUIRED
-
-
 # Work out where our certificates are.
 cert_loc = path.join(path.dirname(__file__), '..', 'certs.pem')
 
+def wrap_socket(sock, server_hostname):
+    """
+    A vastly simplified SSL wrapping function. We'll probably extend this to
+    do more things later.
+    """
+    global _context
 
-if is_py3: # pragma: no cover
-    def wrap_socket(socket, server_hostname):
-        """
-        A vastly simplified SSL wrapping function. We'll probably extend this to
-        do more things later.
-        """
-        global _context
-
-        if _context is None:  # pragma: no cover
-            _context = _init_context()
-
-        if ssl.HAS_SNI:
-            return _context.wrap_socket(socket, server_hostname=server_hostname)
+    if _context is None:  # pragma: no cover
+        _context = _init_context()
 
-        wrapped = _context.wrap_socket(socket)  # pragma: no cover
-        assert wrapped.selected_npn_protocol() == 'HTTP-draft-09/2.0'
-        return wrapped
-else: # pragma: no cover
-    def wrap_socket(socket, server_hostname):
-        return ssl.wrap_socket(socket, ssl_version=ssl.PROTOCOL_SSLv23,
-            ca_certs=cert_loc, cert_reqs=_verify_mode)
+    # the spec requires SNI support
+    ssl_sock = _context.wrap_socket(sock, server_hostname=server_hostname)
+    # Setting SSLContext.check_hostname to True only verifies that the
+    # post-handshake servername matches that of the certificate. We also need to
+    # check that it matches the requested one.
+    ssl.match_hostname(ssl_sock.getpeercert(), server_hostname)
+    with ignore_missing():
+        assert ssl_sock.selected_npn_protocol() == NPN_PROTOCOL
+    return ssl_sock
 
 
-def _init_context(): # pragma: no cover
+def _init_context():
     """
     Creates the singleton SSLContext we use.
     """
-    context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+    context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
     context.set_default_verify_paths()
     context.load_verify_locations(cafile=cert_loc)
-    context.verify_mode = _verify_mode
-
-    try:
-        context.set_npn_protocols(SUPPORTED_PROTOCOLS)
-    except (AttributeError, NotImplementedError):  # pragma: no cover
-        pass
+    context.verify_mode = ssl.CERT_REQUIRED
+    context.check_hostname = True
 
-    # We do our best to do better security
-    try:
-        context.options |= ssl.OP_NO_SSLv2
-    except AttributeError:  # pragma: no cover
-        pass
+    with ignore_missing():
+        context.set_npn_protocols(SUPPORTED_NPN_PROTOCOLS)
 
-    try:
-        context.options |= ssl.OP_NO_COMPRESSION
-    except AttributeError:  # pragma: no cover
-        pass
+    # required by the spec
+    context.options |= ssl.OP_NO_COMPRESSION
 
     return context

hyper/httplib_compat.py

@@ -15,7 +15,7 @@
 except ImportError:
     import httplib
 
-import ssl
+from .compat import ssl
 from .http20.tls import wrap_socket
 
 # If there's no NPN support, we're going to drop all support for HTTP/2.0.