diff --git a/CHANGES/7490.feature b/CHANGES/7490.feature new file mode 100644 index 00000000000..7dda94a850f --- /dev/null +++ b/CHANGES/7490.feature @@ -0,0 +1 @@ +Enabled lenient headers for more flexible parsing in the client. -- by :user:`Dreamsorcerer` diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 8b7d48245d2..92a70f5685f 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -20,6 +20,7 @@ from multidict import CIMultiDict as _CIMultiDict, CIMultiDictProxy as _CIMultiD from yarl import URL as _URL from aiohttp import hdrs +from aiohttp.helpers import DEBUG from .http_exceptions import ( BadHttpMessage, @@ -648,6 +649,9 @@ cdef class HttpResponseParser(HttpParser): max_line_size, max_headers, max_field_size, payload_exception, response_with_body, read_until_eof, auto_decompress) + # Use strict parsing on dev mode, so users are warned about broken servers. + if not DEBUG: + cparser.llhttp_set_lenient_headers(self._cparser, 1) cdef object _on_status_complete(self): if self._buf: diff --git a/docs/index.rst b/docs/index.rst index 44c01a7b08c..a171dc1f48b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -118,6 +118,18 @@ Server example: For more information please visit :ref:`aiohttp-client` and :ref:`aiohttp-web` pages. +Development mode +================ + +When writing your code, we recommend enabling Python's +`development mode `_ +(``python -X dev``). In addition to the extra features enabled for asyncio, aiohttp +will: + +- Use a strict parser in the client code (which can help detect malformed responses + from a server). +- Enable some additional checks (resulting in warnings in certain situations). + What's new in aiohttp 3? ======================== diff --git a/setup.cfg b/setup.cfg index 47ffbc5d209..6d50d321811 100644 --- a/setup.cfg +++ b/setup.cfg @@ -130,6 +130,9 @@ addopts = # `pytest-cov`: --cov=aiohttp --cov=tests/ + + # run tests that are not marked with dev_mode + -m "not dev_mode" filterwarnings = error ignore:module 'ssl' has no attribute 'OP_NO_COMPRESSION'. The Python interpreter is compiled against OpenSSL < 1.0.0. Ref. https.//docs.python.org/3/library/ssl.html#ssl.OP_NO_COMPRESSION:UserWarning @@ -153,3 +156,5 @@ minversion = 3.8.2 testpaths = tests/ junit_family=xunit2 xfail_strict = true +markers = + dev_mode: mark test to run in dev mode. diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index e8d38c193bd..d1678296a5b 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -721,6 +721,23 @@ def test_http_response_parser_no_reason(response) -> None: assert msg.reason == "" +def test_http_response_parser_lenient_headers(response) -> None: + messages, upgrade, tail = response.feed_data( + b"HTTP/1.1 200 test\r\nFoo: abc\x01def\r\n\r\n" + ) + msg = messages[0][0] + + assert msg.headers["Foo"] == "abc\x01def" + + +@pytest.mark.dev_mode +def test_http_response_parser_strict_headers(response) -> None: + if isinstance(response, HttpResponseParserPy): + pytest.xfail("Py parser is lenient. May update py-parser later.") + with pytest.raises(http_exceptions.BadHttpMessage): + response.feed_data(b"HTTP/1.1 200 test\r\nFoo: abc\x01def\r\n\r\n") + + def test_http_response_parser_bad(response) -> None: with pytest.raises(http_exceptions.BadHttpMessage): response.feed_data(b"HTT/1\r\n\r\n")