Merge branch 'master' into app-response-types

CHANGES.rst

@@ -14,6 +14,22 @@ Changelog
 
 .. towncrier release notes start
 
+3.4.4 (2018-09-05)
+==================
+
+- Fix installation from sources when compiling toolkit is not available (`#3241 <https://github.com/aio-libs/aiohttp/pull/3241>`_)
+
+3.4.3 (2018-09-04)
+==================
+
+- Add ``app.pre_frozen`` state to properly handle startup signals in sub-applications. (`#3237 <https://github.com/aio-libs/aiohttp/pull/3237>`_)
+
+
+3.4.2 (2018-09-01)
+==================
+
+- Fix ``iter_chunks`` type annotation (`#3230 <https://github.com/aio-libs/aiohttp/pull/3230>`_)
+
 3.4.1 (2018-08-28)
 ==================
 

CHANGES/3177.feature

@@ -0,0 +1 @@
+Make ``request.url`` accessible when transport is closed.

CHANGES/3233.bugfix

@@ -0,0 +1 @@
+Don't uppercase HTTP method in parser

CHANGES/3237.bugfix

@@ -0,0 +1 @@
+Add ``app.pre_frozen`` state to properly handle startup signals in sub-applications.

CHANGES/3239.bugfix

@@ -0,0 +1 @@
+Enhanced parsing and validation of helpers.BasicAuth.decode.

aiohttp/helpers.py

@@ -95,21 +95,30 @@ def __new__(cls, login: str,
     @classmethod
     def decode(cls, auth_header: str, encoding: str='latin1') -> 'BasicAuth':
         """Create a BasicAuth object from an Authorization HTTP header."""
-        split = auth_header.strip().split(' ')
-        if len(split) == 2:
-            if split[0].strip().lower() != 'basic':
-                raise ValueError('Unknown authorization method %s' % split[0])
-            to_decode = split[1]
-        else:
+        try:
+            auth_type, encoded_credentials = auth_header.split(' ', 1)
+        except ValueError:
             raise ValueError('Could not parse authorization header.')
 
+        if auth_type.lower() != 'basic':
+            raise ValueError('Unknown authorization method %s' % auth_type)
+
         try:
-            username, _, password = base64.b64decode(
-                to_decode.encode('ascii')
-            ).decode(encoding).partition(':')
+            decoded = base64.b64decode(
+                encoded_credentials.encode('ascii'), validate=True
+            ).decode(encoding)
         except binascii.Error:
             raise ValueError('Invalid base64 encoding.')
 
+        try:
+            # RFC 2617 HTTP Authentication
+            # https://www.ietf.org/rfc/rfc2617.txt
+            # the colon must be present, but the username and password may be
+            # otherwise blank.
+            username, password = decoded.split(':', 1)
+        except ValueError:
+            raise ValueError('Invalid credentials.')
+
         return cls(username, password, encoding=encoding)
 
     @classmethod

aiohttp/http_parser.py

@@ -371,7 +371,6 @@ def parse_message(self, lines):
                 'Status line is too long', self.max_line_size, len(path))
 
         # method
-        method = method.upper()
         if not METHRE.match(method):
             raise BadStatusLine(method)
 

aiohttp/streams.py

@@ -37,9 +37,16 @@ async def __anext__(self) -> bytes:
         return rv
 
 
-class ChunkTupleAsyncStreamIterator(AsyncStreamIterator):
-    async def __anext__(self) -> bytes:
-        rv = await self.read_func()
+class ChunkTupleAsyncStreamIterator:
+
+    def __init__(self, stream: 'StreamReader') -> None:
+        self._stream = stream
+
+    def __aiter__(self) -> 'ChunkTupleAsyncStreamIterator':
+        return self
+
+    async def __anext__(self) -> Tuple[bytes, bool]:
+        rv = await self._stream.readchunk()
         if rv == (b'', False):
             raise StopAsyncIteration  # NOQA
         return rv
@@ -72,7 +79,7 @@ def iter_chunks(self) -> ChunkTupleAsyncStreamIterator:
 
         Python-3.5 available for Python 3.5+ only
         """
-        return ChunkTupleAsyncStreamIterator(self.readchunk)  # type: ignore
+        return ChunkTupleAsyncStreamIterator(self)  # type: ignore
 
 
 class StreamReader(AsyncStreamReaderMixin):

aiohttp/web_app.py

@@ -49,7 +49,7 @@ class Application(MutableMapping):
     ATTRS = frozenset([
         'logger', '_debug', '_router', '_loop', '_handler_args',
         '_middlewares', '_middlewares_handlers', '_run_middlewares',
-        '_state', '_frozen', '_subapps',
+        '_state', '_frozen', '_pre_frozen', '_subapps',
         '_on_response_prepare', '_on_startup', '_on_shutdown',
         '_on_cleanup', '_client_max_size', '_cleanup_ctx'])
 
@@ -88,6 +88,7 @@ def __init__(self, *,
 
         self._state = {}  # type: Mapping
         self._frozen = False
+        self._pre_frozen = False
         self._subapps = []  # type: _Subapps
 
         self._on_response_prepare = Signal(self)  # type: _RespPrepareSignal
@@ -166,14 +167,14 @@ def _set_loop(self, loop):
             subapp._set_loop(loop)
 
     @property
-    def frozen(self) -> bool:
-        return self._frozen
+    def pre_frozen(self) -> bool:
+        return self._pre_frozen
 
-    def freeze(self) -> None:
-        if self._frozen:
+    def pre_freeze(self) -> None:
+        if self._pre_frozen:
             return
 
-        self._frozen = True
+        self._pre_frozen = True
         self._middlewares.freeze()
         self._router.freeze()
         self._on_response_prepare.freeze()
@@ -191,10 +192,23 @@ def freeze(self) -> None:
         self._run_middlewares = True if self.middlewares else False
 
         for subapp in self._subapps:
-            subapp.freeze()
+            subapp.pre_freeze()
             self._run_middlewares =\
                 self._run_middlewares or subapp._run_middlewares
 
+    @property
+    def frozen(self) -> bool:
+        return self._frozen
+
+    def freeze(self) -> None:
+        if self._frozen:
+            return
+
+        self.pre_freeze()
+        self._frozen = True
+        for subapp in self._subapps:
+            subapp.freeze()
+
     @property
     def debug(self) -> bool:
         return self._debug
@@ -228,7 +242,7 @@ def add_subapp(self, prefix: str, subapp: 'Application'):
         self.router.register_resource(resource)
         self._reg_subapp_signals(subapp)
         self._subapps.append(subapp)
-        subapp.freeze()
+        subapp.pre_freeze()
         if self._loop is not None:
             subapp._set_loop(self._loop)
         return resource

aiohttp/web_request.py

@@ -78,7 +78,8 @@ class BaseRequest(collections.MutableMapping, HeadersMixin):
     ATTRS = HeadersMixin.ATTRS | frozenset([
         '_message', '_protocol', '_payload_writer', '_payload', '_headers',
         '_method', '_version', '_rel_url', '_post', '_read_bytes',
-        '_state', '_cache', '_task', '_client_max_size', '_loop'])
+        '_state', '_cache', '_task', '_client_max_size', '_loop',
+        '_transport_sslcontext', '_transport_peername'])
 
     def __init__(self, message, payload, protocol, payload_writer, task,
                  loop,
@@ -105,6 +106,10 @@ def __init__(self, message, payload, protocol, payload_writer, task,
         self._client_max_size = client_max_size
         self._loop = loop
 
+        transport = self._protocol.transport
+        self._transport_sslcontext = transport.get_extra_info('sslcontext')
+        self._transport_peername = transport.get_extra_info('peername')
+
         if scheme is not None:
             self._cache['scheme'] = scheme
         if host is not None:
@@ -292,7 +297,7 @@ def scheme(self) -> str:
 
         'http' or 'https'.
         """
-        if self.transport.get_extra_info('sslcontext'):
+        if self._transport_sslcontext:
             return 'https'
         else:
             return 'http'
@@ -338,13 +343,10 @@ def remote(self) -> Optional[str]:
         - overridden value by .clone(remote=new_remote) call.
         - peername of opened socket
         """
-        if self.transport is None:
-            return None
-        peername = self.transport.get_extra_info('peername')
-        if isinstance(peername, (list, tuple)):
-            return peername[0]
+        if isinstance(self._transport_peername, (list, tuple)):
+            return self._transport_peername[0]
         else:
-            return peername
+            return self._transport_peername
 
     @reify
     def url(self) -> URL:

requirements/ci-wheel.txt

@@ -1,5 +1,5 @@
 -r flake.txt
-attrs==18.1.0
+attrs==18.2.0
 async-generator==1.10
 async-timeout==3.0.0
 brotlipy==0.7.0
@@ -10,11 +10,11 @@ cython==0.28.5
 gunicorn==19.8.1
 pyflakes==2.0.0
 multidict==4.3.1
-pytest==3.7.2
+pytest==3.7.4
 pytest-cov==2.5.1
 pytest-mock==1.10.0
-pytest-xdist==1.22.5
-tox==3.1.3
+pytest-xdist==1.23.0
+tox==3.2.1
 twine==1.11.0
 yarl==1.2.6
 

requirements/dev.txt

@@ -3,4 +3,4 @@
 ipdb==0.11
 pytest-sugar==0.9.1
 ipython==6.5.0
-cherry_picker==1.2.0; python_version>="3.6"
+cherry_picker==1.2.1; python_version>="3.6"

requirements/doc.txt

@@ -1,4 +1,4 @@
-sphinx==1.7.7
+sphinx==1.7.8
 sphinxcontrib-asyncio==0.2.0
 pygments>=2.1
 aiohttp-theme==0.1.4

requirements/wheel.txt

@@ -1,3 +1,3 @@
 cython==0.28.5
-pytest==3.7.2
+pytest==3.7.4
 twine==1.11.0

setup.cfg

@@ -34,6 +34,8 @@ branch = True
 source = aiohttp, tests
 omit = site-packages
 
+[mypy]
+incremental = false
 
 [mypy-pytest]
 ignore_missing_imports = true

setup.py

@@ -72,7 +72,7 @@ def run(self):
     def build_extension(self, ext):
         try:
             build_ext.build_extension(self, ext)
-        except (DistutilsExecError,
+        except (CCompilerError, DistutilsExecError,
                 DistutilsPlatformError, ValueError):
             raise BuildFailed()
 

tests/test_helpers.py

@@ -1,4 +1,5 @@
 import asyncio
+import base64
 import datetime
 import gc
 import os
@@ -83,8 +84,12 @@ def test_basic_auth4():
     assert auth.encode() == 'Basic bmtpbTpwd2Q='
 
 
-def test_basic_auth_decode():
-    auth = helpers.BasicAuth.decode('Basic bmtpbTpwd2Q=')
+@pytest.mark.parametrize('header', (
+    'Basic bmtpbTpwd2Q=',
+    'basic bmtpbTpwd2Q=',
+))
+def test_basic_auth_decode(header):
+    auth = helpers.BasicAuth.decode(header)
     assert auth.login == 'nkim'
     assert auth.password == 'pwd'
 
@@ -104,6 +109,33 @@ def test_basic_auth_decode_bad_base64():
         helpers.BasicAuth.decode('Basic bmtpbTpwd2Q')
 
 
+@pytest.mark.parametrize('header', ('Basic ???', 'Basic   '))
+def test_basic_auth_decode_illegal_chars_base64(header):
+    with pytest.raises(ValueError, match='Invalid base64 encoding.'):
+        helpers.BasicAuth.decode(header)
+
+
+def test_basic_auth_decode_invalid_credentials():
+    with pytest.raises(ValueError, match='Invalid credentials.'):
+        header = 'Basic {}'.format(base64.b64encode(b'username').decode())
+        helpers.BasicAuth.decode(header)
+
+
+@pytest.mark.parametrize('credentials, expected_auth', (
+    (':', helpers.BasicAuth(
+        login='', password='', encoding='latin1')),
+    ('username:', helpers.BasicAuth(
+        login='username', password='', encoding='latin1')),
+    (':password', helpers.BasicAuth(
+        login='', password='password', encoding='latin1')),
+    ('username:password', helpers.BasicAuth(
+        login='username', password='password', encoding='latin1')),
+))
+def test_basic_auth_decode_blank_username(credentials, expected_auth):
+    header = 'Basic {}'.format(base64.b64encode(credentials.encode()).decode())
+    assert helpers.BasicAuth.decode(header) == expected_auth
+
+
 def test_basic_auth_from_url():
     url = URL('http://user:pass@example.com')
     auth = helpers.BasicAuth.from_url(url)