diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 852f352e067..33b14a68849 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -238,6 +238,13 @@ jobs: run: >- # `exit 1` makes sure that the job remains red with flaky runs pytest --no-cov -vvvvv --lf && exit 1 shell: bash + - name: Run dev_mode tests + env: + COLOR: yes + AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }} + PIP_USER: 1 + run: python -X dev -m pytest -m dev_mode --cov-append + shell: bash - name: Turn coverage into xml env: COLOR: 'yes' diff --git a/CHANGES/7490.feature b/CHANGES/7490.feature new file mode 100644 index 00000000000..7dda94a850f --- /dev/null +++ b/CHANGES/7490.feature @@ -0,0 +1 @@ +Enabled lenient headers for more flexible parsing in the client. -- by :user:`Dreamsorcerer` diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 8b7d48245d2..92a70f5685f 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -20,6 +20,7 @@ from multidict import CIMultiDict as _CIMultiDict, CIMultiDictProxy as _CIMultiD from yarl import URL as _URL from aiohttp import hdrs +from aiohttp.helpers import DEBUG from .http_exceptions import ( BadHttpMessage, @@ -648,6 +649,9 @@ cdef class HttpResponseParser(HttpParser): max_line_size, max_headers, max_field_size, payload_exception, response_with_body, read_until_eof, auto_decompress) + # Use strict parsing on dev mode, so users are warned about broken servers. + if not DEBUG: + cparser.llhttp_set_lenient_headers(self._cparser, 1) cdef object _on_status_complete(self): if self._buf: diff --git a/docs/index.rst b/docs/index.rst index a8b99234ae7..4b8a6263a94 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -118,6 +118,18 @@ Server example: For more information please visit :ref:`aiohttp-client` and :ref:`aiohttp-web` pages. +Development mode +================ + +When writing your code, we recommend enabling Python's +`development mode `_ +(``python -X dev``). In addition to the extra features enabled for asyncio, aiohttp +will: + +- Use a strict parser in the client code (which can help detect malformed responses + from a server). +- Enable some additional checks (resulting in warnings in certain situations). + What's new in aiohttp 3? ======================== diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 498103a0b73..367cf14da95 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -730,6 +730,23 @@ def test_http_response_parser_no_reason(response: Any) -> None: assert msg.reason == "" +def test_http_response_parser_lenient_headers(response: Any) -> None: + messages, upgrade, tail = response.feed_data( + b"HTTP/1.1 200 test\r\nFoo: abc\x01def\r\n\r\n" + ) + msg = messages[0][0] + + assert msg.headers["Foo"] == "abc\x01def" + + +@pytest.mark.dev_mode +def test_http_response_parser_strict_headers(response: Any) -> None: + if isinstance(response, HttpResponseParserPy): + pytest.xfail("Py parser is lenient. May update py-parser later.") + with pytest.raises(http_exceptions.BadHttpMessage): + response.feed_data(b"HTTP/1.1 200 test\r\nFoo: abc\x01def\r\n\r\n") + + def test_http_response_parser_bad(response: Any) -> None: with pytest.raises(http_exceptions.BadHttpMessage): response.feed_data(b"HTT/1\r\n\r\n")