From 2b9e24c3d331cd8efacc18b983a3bcab33b3e43b Mon Sep 17 00:00:00 2001 From: Michiel Vanderlee <918128+mvanderlee@users.noreply.github.com> Date: Wed, 4 Aug 2021 23:52:33 -0300 Subject: [PATCH 1/3] Update default max header field size to 65536 --- aiohttp/_http_parser.pyx | 6 +++--- aiohttp/http_parser.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index c24e31057a8..b2c63a43c96 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -328,7 +328,7 @@ cdef class HttpParser: object protocol, object loop, int limit, object timer=None, size_t max_line_size=8190, size_t max_headers=32768, - size_t max_field_size=8190, payload_exception=None, + size_t max_field_size=65536, payload_exception=None, bint response_with_body=True, bint read_until_eof=False, bint auto_decompress=True): cparser.http_parser_init(self._cparser, mode) @@ -564,7 +564,7 @@ cdef class HttpRequestParser(HttpParser): def __init__(self, protocol, loop, int limit, timer=None, size_t max_line_size=8190, size_t max_headers=32768, - size_t max_field_size=8190, payload_exception=None, + size_t max_field_size=65536, payload_exception=None, bint response_with_body=True, bint read_until_eof=False, ): self._init(cparser.HTTP_REQUEST, protocol, loop, limit, timer, @@ -592,7 +592,7 @@ cdef class HttpResponseParser(HttpParser): def __init__(self, protocol, loop, int limit, timer=None, size_t max_line_size=8190, size_t max_headers=32768, - size_t max_field_size=8190, payload_exception=None, + size_t max_field_size=65536, payload_exception=None, bint response_with_body=True, bint read_until_eof=False, bint auto_decompress=True ): diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index 1045b6c0926..a818a1e2b81 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -125,7 +125,7 @@ def __init__( self, max_line_size: int = 8190, max_headers: int = 32768, - max_field_size: int = 8190, + max_field_size: int = 65536, ) -> None: self.max_line_size = max_line_size self.max_headers = max_headers @@ -222,7 +222,7 @@ def __init__( limit: int, max_line_size: int = 8190, max_headers: int = 32768, - max_field_size: int = 8190, + max_field_size: int = 65536, timer: Optional[BaseTimerContext] = None, code: Optional[int] = None, method: Optional[str] = None, From 89bc517d27d8b93a140647c482da01ce67aa587f Mon Sep 17 00:00:00 2001 From: Michiel Vanderlee <918128+mvanderlee@users.noreply.github.com> Date: Thu, 5 Aug 2021 10:01:25 -0300 Subject: [PATCH 2/3] Make the default header size limits configurable via os environments --- aiohttp/_http_parser.pyx | 17 +++++++++++------ aiohttp/http_parser.py | 16 ++++++++++------ 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index b2c63a43c96..259c778fbd9 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -18,6 +18,7 @@ from libc.string cimport memcpy from multidict import CIMultiDict as _CIMultiDict, CIMultiDictProxy as _CIMultiDictProxy from yarl import URL as _URL +from os import getenv from aiohttp import hdrs @@ -49,6 +50,10 @@ from aiohttp cimport _find_header DEF DEFAULT_FREELIST_SIZE = 250 +DEFAULT_MAX_LINE_SIZE = int(os.getenv('AIOHTTP_DEFAULT_MAX_LINE_SIZE', 8190)) +DEFAULT_MAX_HEADERS = int(os.getenv('AIOHTTP_DEFAULT_MAX_HEADERS', 32768)) +DEFAULT_MAX_FIELD_SIZE = int(os.getenv('AIOHTTP_DEFAULT_MAX_FIELD_SIZE', 65536)) + cdef extern from "Python.h": int PyByteArray_Resize(object, Py_ssize_t) except -1 Py_ssize_t PyByteArray_Size(object) except -1 @@ -327,8 +332,8 @@ cdef class HttpParser: cdef _init(self, cparser.http_parser_type mode, object protocol, object loop, int limit, object timer=None, - size_t max_line_size=8190, size_t max_headers=32768, - size_t max_field_size=65536, payload_exception=None, + size_t max_line_size=DEFAULT_MAX_LINE_SIZE, size_t max_headers=DEFAULT_MAX_HEADERS, + size_t max_field_size=DEFAULT_MAX_FIELD_SIZE, payload_exception=None, bint response_with_body=True, bint read_until_eof=False, bint auto_decompress=True): cparser.http_parser_init(self._cparser, mode) @@ -563,8 +568,8 @@ cdef class HttpParser: cdef class HttpRequestParser(HttpParser): def __init__(self, protocol, loop, int limit, timer=None, - size_t max_line_size=8190, size_t max_headers=32768, - size_t max_field_size=65536, payload_exception=None, + size_t max_line_size=DEFAULT_MAX_LINE_SIZE, size_t max_headers=DEFAULT_MAX_HEADERS, + size_t max_field_size=DEFAULT_MAX_FIELD_SIZE, payload_exception=None, bint response_with_body=True, bint read_until_eof=False, ): self._init(cparser.HTTP_REQUEST, protocol, loop, limit, timer, @@ -591,8 +596,8 @@ cdef class HttpRequestParser(HttpParser): cdef class HttpResponseParser(HttpParser): def __init__(self, protocol, loop, int limit, timer=None, - size_t max_line_size=8190, size_t max_headers=32768, - size_t max_field_size=65536, payload_exception=None, + size_t max_line_size=DEFAULT_MAX_LINE_SIZE, size_t max_headers=DEFAULT_MAX_HEADERS, + size_t max_field_size=DEFAULT_MAX_FIELD_SIZE, payload_exception=None, bint response_with_body=True, bint read_until_eof=False, bint auto_decompress=True ): diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index a818a1e2b81..12e03821e82 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -1,6 +1,7 @@ import abc import asyncio import collections +import os import re import string import zlib @@ -71,6 +72,9 @@ VERSRE: Final[Pattern[str]] = re.compile(r"HTTP/(\d+).(\d+)") HDRRE: Final[Pattern[bytes]] = re.compile(rb"[\x00-\x1F\x7F()<>@,;:\[\]={} \t\\\\\"]") +DEFAULT_MAX_LINE_SIZE = int(os.getenv('AIOHTTP_DEFAULT_MAX_LINE_SIZE', 8190)) +DEFAULT_MAX_HEADERS = int(os.getenv('AIOHTTP_DEFAULT_MAX_HEADERS', 32768)) +DEFAULT_MAX_FIELD_SIZE = int(os.getenv('AIOHTTP_DEFAULT_MAX_FIELD_SIZE', 65536)) class RawRequestMessage(NamedTuple): method: str @@ -123,9 +127,9 @@ class ChunkState(IntEnum): class HeadersParser: def __init__( self, - max_line_size: int = 8190, - max_headers: int = 32768, - max_field_size: int = 65536, + max_line_size: int = DEFAULT_MAX_LINE_SIZE, + max_headers: int = DEFAULT_MAX_HEADERS, + max_field_size: int = DEFAULT_MAX_FIELD_SIZE, ) -> None: self.max_line_size = max_line_size self.max_headers = max_headers @@ -220,9 +224,9 @@ def __init__( protocol: BaseProtocol, loop: asyncio.AbstractEventLoop, limit: int, - max_line_size: int = 8190, - max_headers: int = 32768, - max_field_size: int = 65536, + max_line_size: int = DEFAULT_MAX_LINE_SIZE, + max_headers: int = DEFAULT_MAX_HEADERS, + max_field_size: int = DEFAULT_MAX_FIELD_SIZE, timer: Optional[BaseTimerContext] = None, code: Optional[int] = None, method: Optional[str] = None, From 3816028af7fd16038200ea63db3df8074fb5c576 Mon Sep 17 00:00:00 2001 From: Michiel Vanderlee <918128+mvanderlee@users.noreply.github.com> Date: Thu, 5 Aug 2021 10:05:25 -0300 Subject: [PATCH 3/3] Add changelog --- CHANGES/2304.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 CHANGES/2304.bugfix diff --git a/CHANGES/2304.bugfix b/CHANGES/2304.bugfix new file mode 100644 index 00000000000..1d17ee84e82 --- /dev/null +++ b/CHANGES/2304.bugfix @@ -0,0 +1 @@ +Set more reasonable default header limits and make them configurable via environment variables.