From 188852b8d9ff082ce7793f3f8d03128cc62f3ac1 Mon Sep 17 00:00:00 2001 From: Stefan Date: Wed, 27 Apr 2022 14:12:17 +0200 Subject: [PATCH] #2304 add max_line_site, max_headers and max_field_size arguments to ClientSession --- CHANGES/2304.feature | 1 + aiohttp/client.py | 30 ++++++ aiohttp/client_proto.py | 6 ++ docs/client_reference.rst | 22 ++++- tests/test_client_functional.py | 170 ++++++++++++++++++++++++++++++++ 5 files changed, 227 insertions(+), 2 deletions(-) create mode 100644 CHANGES/2304.feature diff --git a/CHANGES/2304.feature b/CHANGES/2304.feature new file mode 100644 index 00000000000..a8c844896fa --- /dev/null +++ b/CHANGES/2304.feature @@ -0,0 +1 @@ +Support setting response header parameters max_line_size, max_headers and max_field_size. diff --git a/aiohttp/client.py b/aiohttp/client.py index d05689ca6b7..8918786791e 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -191,6 +191,9 @@ class ClientSession: "_ws_response_class", "_trace_configs", "_read_bufsize", + "_max_line_size", + "_max_headers", + "_max_field_size", ) def __init__( @@ -218,6 +221,9 @@ def __init__( requote_redirect_url: bool = True, trace_configs: Optional[List[TraceConfig]] = None, read_bufsize: int = 2**16, + max_line_size: int = 8190, + max_headers: int = 32768, + max_field_size: int = 8190, ) -> None: if base_url is None or isinstance(base_url, URL): self._base_url: Optional[URL] = base_url @@ -266,6 +272,9 @@ def __init__( self._trust_env = trust_env self._requote_redirect_url = requote_redirect_url self._read_bufsize = read_bufsize + self._max_line_size = max_line_size + self._max_headers = max_headers + self._max_field_size = max_field_size # Convert to list of tuples if headers: @@ -351,6 +360,9 @@ async def _request( proxy_headers: Optional[LooseHeaders] = None, trace_request_ctx: Optional[SimpleNamespace] = None, read_bufsize: Optional[int] = None, + max_line_size: Optional[int] = None, + max_headers: Optional[int] = None, + max_field_size: Optional[int] = None, ) -> ClientResponse: # NOTE: timeout clamps existing connect and read timeouts. We cannot @@ -411,6 +423,15 @@ async def _request( if read_bufsize is None: read_bufsize = self._read_bufsize + if max_line_size is None: + max_line_size = self._max_line_size + + if max_headers is None: + max_headers = self._max_headers + + if max_field_size is None: + max_field_size = self._max_field_size + traces = [ Trace( self, @@ -516,6 +537,9 @@ async def _request( read_timeout=real_timeout.sock_read, read_bufsize=read_bufsize, timeout_ceil_threshold=self._connector._timeout_ceil_threshold, + max_line_size=max_line_size, + max_headers=max_headers, + max_field_size=max_field_size, ) try: @@ -1193,6 +1217,9 @@ def request( version: HttpVersion = http.HttpVersion11, connector: Optional[BaseConnector] = None, read_bufsize: Optional[int] = None, + max_line_size: int = 8190, + max_headers: int = 32768, + max_field_size: int = 8190, ) -> _SessionRequestContextManager: """Constructs and sends a request. @@ -1263,6 +1290,9 @@ def request( proxy=proxy, proxy_auth=proxy_auth, read_bufsize=read_bufsize, + max_line_size=max_line_size, + max_headers=max_headers, + max_field_size=max_field_size, ), session, ) diff --git a/aiohttp/client_proto.py b/aiohttp/client_proto.py index 0e6c414ea7d..fa8215c0cd8 100644 --- a/aiohttp/client_proto.py +++ b/aiohttp/client_proto.py @@ -153,6 +153,9 @@ def set_response_params( read_timeout: Optional[float] = None, read_bufsize: int = 2**16, timeout_ceil_threshold: float = 5, + max_line_size: int = 8190, + max_headers: int = 32768, + max_field_size: int = 8190, ) -> None: self._skip_payload = skip_payload @@ -170,6 +173,9 @@ def set_response_params( response_with_body=not skip_payload, read_until_eof=read_until_eof, auto_decompress=auto_decompress, + max_line_size=max_line_size, + max_headers=max_headers, + max_field_size=max_field_size, ) if self._tail: diff --git a/docs/client_reference.rst b/docs/client_reference.rst index c423e54458c..426eb9bd491 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -52,7 +52,10 @@ The client session supports the context manager protocol for self closing. read_bufsize=2**16, \ requote_redirect_url=False, \ trust_env=False, \ - trace_configs=None) + trace_configs=None, \ + max_line_size=8190, \ + max_headers=32768, \ + max_field_size=8190) The class for creating client sessions and making requests. @@ -201,6 +204,12 @@ The client session supports the context manager protocol for self closing. disabling. See :ref:`aiohttp-client-tracing-reference` for more information. + :param max_line_size: The maximum length allowed for the HTTP response reason field. + + :param max_headers: The maximum number of response headers allowed. + + :param max_field_size: The maximum length allowed for response header values. + .. attribute:: closed ``True`` if the session has been closed, ``False`` otherwise. @@ -338,7 +347,10 @@ The client session supports the context manager protocol for self closing. proxy=None, proxy_auth=None,\ timeout=sentinel, ssl=None, \ verify_ssl=None, fingerprint=None, \ - ssl_context=None, proxy_headers=None) + ssl_context=None, proxy_headers=None, \ + max_line_size=8190, \ + max_headers=32768, \ + max_field_size=8190) :async-with: :coroutine: :noindex: @@ -510,6 +522,12 @@ The client session supports the context manager protocol for self closing. .. versionadded:: 3.0 + :param max_line_size: The maximum length allowed for the HTTP response reason field. + + :param max_headers: The maximum number of response headers allowed. + + :param max_field_size: The maximum length allowed for response header values. + :return ClientResponse: a :class:`client response ` object. diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index b77bd516788..52f311751da 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -3028,3 +3028,173 @@ async def handler(request): assert resp.status == 200 assert await resp.text() == "ok" assert resp.headers["Content-Type"] == "text/plain; charset=utf-8" + + +async def test_max_field_size_session_default(aiohttp_client: Any) -> None: + async def handler(request): + return web.Response(headers={"Custom": "x" * 8190}) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app) + + async with await client.get("/") as resp: + assert resp.headers["Custom"] == "x" * 8190 + + +async def test_max_field_size_session_default_fail(aiohttp_client: Any) -> None: + async def handler(request): + return web.Response(headers={"Custom": "x" * 8191}) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app) + with pytest.raises(aiohttp.ClientResponseError): + await client.get("/") + + +async def test_max_field_size_session_explicit(aiohttp_client: Any) -> None: + async def handler(request): + return web.Response(headers={"Custom": "x" * 8191}) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app, max_field_size=8191) + + async with await client.get("/") as resp: + assert resp.headers["Custom"] == "x" * 8191 + + +async def test_max_field_size_request_explicit(aiohttp_client: Any) -> None: + async def handler(request): + return web.Response(headers={"Custom": "x" * 8191}) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app) + + async with await client.get("/", max_field_size=8191) as resp: + assert resp.headers["Custom"] == "x" * 8191 + + +async def test_max_line_size_session_default(aiohttp_client: Any) -> None: + async def handler(request): + return web.Response(status=200, reason="x" * 8190) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app) + + async with await client.get("/") as resp: + assert resp.reason == "x" * 8190 + + +async def test_max_line_size_session_default_fail(aiohttp_client: Any) -> None: + async def handler(request): + return web.Response(status=200, reason="x" * 8192) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app) + with pytest.raises(aiohttp.ClientResponseError): + await client.get("/") + + +async def test_max_line_size_session_explicit(aiohttp_client: Any) -> None: + async def handler(request): + return web.Response(status=200, reason="x" * 8191) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app, max_line_size=8191) + + async with await client.get("/") as resp: + assert resp.reason == "x" * 8191 + + +async def test_max_line_size_request_explicit(aiohttp_client: Any) -> None: + async def handler(request): + return web.Response(status=200, reason="x" * 8191) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app) + + async with await client.get("/", max_line_size=8191) as resp: + assert resp.reason == "x" * 8191 + + +async def test_max_headers_session_default(aiohttp_client: Any) -> None: + async def handler(request): + # generate 32764 headers: + # 32768 (max_headers default) minus 4 headers which are set implicitly + # 'Content-Length', 'Content-Type', 'Date' and 'Server' + headers = MultiDict() + for x in range(32764): + headers.add(f"x-header-{x}", str(x)) + return web.Response(headers=headers) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app) + + async with await client.get("/") as resp: + assert len(resp.headers) == 32768 + + +@pytest.mark.xfail +async def test_max_headers_session_default_fail(aiohttp_client: Any) -> None: + async def handler(request): + headers = MultiDict() + for x in range(32769): + headers.add(f"x-header-{x}", str(x)) + return web.Response(headers=headers) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app) + + with pytest.raises(aiohttp.ClientResponseError): + await client.get("/") + + +async def test_max_headers_session_explicit(aiohttp_client: Any) -> None: + async def handler(request): + headers = MultiDict() + for x in range(32765): + headers.add(f"x-header-{x}", str(x)) + return web.Response(headers=headers) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app, max_headers=32769) + + async with await client.get("/") as resp: + assert len(resp.headers) == 32769 + + +async def test_max_headers_request_explicit(aiohttp_client: Any) -> None: + async def handler(request): + headers = MultiDict() + for x in range(32765): + headers.add(f"x-header-{x}", str(x)) + return web.Response(headers=headers) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app) + + async with await client.get("/", max_headers=32769) as resp: + assert len(resp.headers) == 32769