diff --git a/tempesta_fw/t/functional/health_monitoring/test_health_monitor.py b/tempesta_fw/t/functional/health_monitoring/test_health_monitor.py index f9b3a4ec0f..5759334a39 100644 --- a/tempesta_fw/t/functional/health_monitoring/test_health_monitor.py +++ b/tempesta_fw/t/functional/health_monitoring/test_health_monitor.py @@ -29,7 +29,7 @@ def prepare(self): self.tester.current_chain = copy.copy(self.chain) self.tester.recieved_chain = deproxy.MessageChain.empty() self.client.clear() - self.client.set_request(self.tester.current_chain.request) + self.client.set_request(self.tester.current_chain) def check_transition(self, messages): expected = None diff --git a/tempesta_fw/t/functional/helpers/__init__.py b/tempesta_fw/t/functional/helpers/__init__.py index 9bdcd2099b..be67731d0c 100644 --- a/tempesta_fw/t/functional/helpers/__init__.py +++ b/tempesta_fw/t/functional/helpers/__init__.py @@ -1,4 +1,4 @@ __all__ = ['tf_cfg', 'deproxy', 'nginx', 'tempesta', 'error', 'flacky', - 'analyzer', 'stateful', 'dmesg'] + 'analyzer', 'stateful', 'dmesg', 'wrk'] # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 diff --git a/tempesta_fw/t/functional/helpers/chains.py b/tempesta_fw/t/functional/helpers/chains.py index 23174cd28d..5b3caa89b4 100644 --- a/tempesta_fw/t/functional/helpers/chains.py +++ b/tempesta_fw/t/functional/helpers/chains.py @@ -33,6 +33,37 @@ def make_502_expected(): ) return response +def response_500(): + date = deproxy.HttpMessage.date_time_string() + headers = [ + 'Date: %s' % date, + 'Content-Length: 0', + 'Connection: keep-alive' + ] + response = deproxy.Response.create(status=500, headers=headers) + return response + +def response_403(date=None, connection=None): + if date is None: + date = deproxy.HttpMessage.date_time_string() + headers = ['Content-Length: 0'] + if connection != None: + headers.append('Connection: %s' % connection) + + return deproxy.Response.create(status=403, headers=headers, + date=date, body='') + +def response_400(date=None, connection=None): + if date is None: + date = deproxy.HttpMessage.date_time_string() + headers = ['Content-Length: 0'] + if connection != None: + headers.append('Connection: %s' % connection) + + resp = deproxy.Response.create(status=400, headers=headers, + date=date, body='') + return resp + def base(uri='/', method='GET', forward=True, date=None): """Base message chain. Looks like simple Curl request to Tempesta and response for it. @@ -85,7 +116,6 @@ def base(uri='/', method='GET', forward=True, date=None): common_resp_date = date # response body common_resp_body = '' - common_resp_body_void = False # common part of response headers common_resp_headers = [ 'Connection: keep-alive', @@ -122,8 +152,6 @@ def base(uri='/', method='GET', forward=True, date=None): ] if method == "GET": common_resp_body = sample_body - else: - common_resp_body_void = True elif method == "POST": common_req_headers += [ @@ -165,12 +193,13 @@ def base(uri='/', method='GET', forward=True, date=None): uri=uri, body=common_req_body ) + tempesta_resp = deproxy.Response.create( status=common_resp_code, headers=common_resp_headers + tempesta_resp_headers_addn, date=common_resp_date, body=common_resp_body, - body_void=common_resp_body_void + method=method, ) if forward: @@ -180,12 +209,13 @@ def base(uri='/', method='GET', forward=True, date=None): uri=uri, body=common_req_body ) + backend_resp = deproxy.Response.create( status=common_resp_code, headers=common_resp_headers + backend_resp_headers_addn, date=common_resp_date, body=common_resp_body, - body_void=common_resp_body_void + method=method, ) else: tempesta_req = None @@ -197,7 +227,6 @@ def base(uri='/', method='GET', forward=True, date=None): server_response=backend_resp) return copy.copy(base_chain) - def base_chunked(uri='/'): """Same as chains.base(), but returns a copy of message chain with chunked body. diff --git a/tempesta_fw/t/functional/helpers/control.py b/tempesta_fw/t/functional/helpers/control.py index eecee25e75..1f1fc84a64 100644 --- a/tempesta_fw/t/functional/helpers/control.py +++ b/tempesta_fw/t/functional/helpers/control.py @@ -127,18 +127,27 @@ def __init__(self, threads=-1, uri='/', ssl=False): Client.__init__(self, binary='wrk', uri=uri, ssl=ssl) self.threads = threads self.script = '' - - def set_script(self, script): - self.script = script + self.local_scriptdir = ''.join([ + os.path.dirname(os.path.realpath(__file__)), + '/../wrk/']) + self.copy_script = True + + def set_script(self, script, content=None): + self.script = script + ".lua" + if content == None: + local_path = ''.join([self.local_scriptdir, self.script]) + local_script_path = os.path.abspath(local_path) + assert os.path.isfile(local_script_path), \ + 'No script found: %s !' % local_script_path + f = open(local_script_path, 'r') + self.files.append((self.script, f.read())) + else: + self.node.copy_file(self.script, content) def append_script_option(self): if not self.script: return - path = ''.join([os.path.dirname(os.path.realpath(__file__)), - '/../wrk/', self.script, '.lua']) - script_path = os.path.abspath(path) - assert os.path.isfile(script_path), \ - 'No script found: %s !' % script_path + script_path = self.workdir + "/" + self.script self.options.append('-s %s' % script_path) def form_command(self): diff --git a/tempesta_fw/t/functional/helpers/deproxy.py b/tempesta_fw/t/functional/helpers/deproxy.py index d71b8c3ca5..5478ed05f9 100644 --- a/tempesta_fw/t/functional/helpers/deproxy.py +++ b/tempesta_fw/t/functional/helpers/deproxy.py @@ -25,7 +25,6 @@ class ParseError(Exception): class IncompliteMessage(ParseError): pass - class HeaderCollection(object): """ A collection class for HTTP Headers. This class combines aspects of a list @@ -207,10 +206,10 @@ def __repr__(self): class HttpMessage(object): __metaclass__ = abc.ABCMeta - def __init__(self, message_text=None, body_parsing=True, body_void=False): + def __init__(self, message_text=None, body_parsing=True, method="GET"): self.msg = '' + self.method = method self.body_parsing = True - self.body_void = body_void # For responses to HEAD requests self.headers = HeaderCollection() self.trailer = HeaderCollection() self.body = '' @@ -222,7 +221,7 @@ def parse_text(self, message_text, body_parsing=True): self.body_parsing = body_parsing stream = StringIO(message_text) self.__parse(stream) - self.__set_str_msg() + self.build_message() def __parse(self, stream): self.parse_firstline(stream) @@ -230,33 +229,36 @@ def __parse(self, stream): self.body = '' self.parse_body(stream) - def __set_str_msg(self): + def build_message(self): self.msg = str(self) @abc.abstractmethod def parse_firstline(self, stream): pass + @abc.abstractmethod + def parse_body(self, stream): + pass + def get_firstline(self): return '' def parse_headers(self, stream): self.headers = HeaderCollection.from_stream(stream) - def parse_body(self, stream): - if self.body_parsing and 'Transfer-Encoding' in self.headers: - enc = self.headers['Transfer-Encoding'] - option = enc.split(',')[-1] # take the last option + def read_encoded_body(self, stream): + """ RFC 7230. 3.3.3 #3 """ + enc = self.headers['Transfer-Encoding'] + option = enc.split(',')[-1] # take the last option - if option.strip().lower() == 'chunked': - self.read_chunked_body(stream) - else: - error.bug('Not implemented!') - elif self.body_parsing and 'Content-Length' in self.headers: - length = int(self.headers['Content-Length']) - self.read_sized_body(stream, length) + if option.strip().lower() == 'chunked': + self.read_chunked_body(stream) else: - self.body = stream.read() + error.bug('Not implemented!') + + def read_rest_body(self, stream): + """ RFC 7230. 3.3.3 #7 """ + self.body = stream.read() def read_chunked_body(self, stream): while True: @@ -278,11 +280,10 @@ def read_chunked_body(self, stream): # Parsing trailer will eat last CRLF self.parse_trailer(stream) - def read_sized_body(self, stream, size): - if self.body_void: - return - if size == 0: - return + def read_sized_body(self, stream): + """ RFC 7230. 3.3.3 #5 """ + size = int(self.headers['Content-Length']) + self.body = stream.read(size) if len(self.body) != size: raise ParseError(("Wrong body size: expect %d but got %d!" @@ -379,6 +380,19 @@ def parse_firstline(self, stream): def get_firstline(self): return ' '.join([self.method, self.uri, self.version]) + def parse_body(self, stream): + """ RFC 7230 3.3.3 """ + # 3.3.3 3 + if 'Transfer-Encoding' in self.headers: + self.read_encoded_body(stream) + return + # 3.3.3 5 + if 'Content-Length' in self.headers: + self.read_sized_body(stream) + return + # 3.3.3 6 + self.body = '' + def __eq__(self, other): return ((self.method == other.method) and (self.version == other.version) @@ -422,6 +436,31 @@ def parse_firstline(self, stream): except: raise ParseError('Invalid Status code!') + def parse_body(self, stream): + """ RFC 7230 3.3.3 """ + # 3.3.3 1 + if self.method == "HEAD": + return + code = int(self.status) + if code >= 100 and code <= 199 or \ + code == 204 or code == 304: + return + # 3.3.3 2 + if self.method == "CONNECT" and code >= 200 and code <= 299: + error.bug('Not implemented!') + return + # 3.3.3 3 + if 'Transfer-Encoding' in self.headers: + self.read_encoded_body(stream) + return + # TODO: check 3.3.3 4 + # 3.3.3 5 + if 'Content-Length' in self.headers: + self.read_sized_body(stream) + return + # 3.3.3 7 + self.read_rest_body(stream) + def get_firstline(self): status = int(self.status) reason = BaseHTTPRequestHandler.responses[status][0] @@ -438,12 +477,12 @@ def __ne__(self, other): @staticmethod def create(status, headers, version='HTTP/1.1', date=False, - srv_version=None, body=None, body_void=False): + srv_version=None, body=None, method='GET'): reason = BaseHTTPRequestHandler.responses first_line = ' '.join([version, str(status), reason[status][0]]) msg = HttpMessage.create(first_line, headers, date=date, srv_version=srv_version, body=body) - return Response(msg, body_void=body_void) + return Response(msg, method=method) #------------------------------------------------------------------------------- # HTTP Client/Server @@ -481,10 +520,10 @@ def run_start(self): def clear(self): self.request_buffer = '' - def set_request(self, request): - if request: - self.request = request - self.request_buffer = request.msg + def set_request(self, message_chain): + if message_chain: + self.request = message_chain.request + self.request_buffer = message_chain.request.msg def set_tester(self, tester): self.tester = tester @@ -503,7 +542,7 @@ def handle_read(self): tf_cfg.dbg(5, self.response_buffer) try: response = Response(self.response_buffer, - body_void=(self.request.method == 'HEAD')) + method=self.request.method) self.response_buffer = self.response_buffer[len(response.msg):] except IncompliteMessage: return @@ -512,6 +551,10 @@ def handle_read(self): '<<<<<\n%s>>>>>' % self.response_buffer)) raise + if len(self.response_buffer) > 0: + # TODO: take care about pipelined case + raise ParseError('Garbage after response end:\n```\n%s\n```\n' % \ + self.response_buffer) if self.tester: self.tester.recieved_response(response) self.response_buffer = '' @@ -529,7 +572,11 @@ def handle_write(self): def handle_error(self): _, v, _ = sys.exc_info() - error.bug('\tDeproxy: Client: %s' % v) + if type(v) == ParseError or type(v) == AssertionError: + raise v + else: + error.bug('\tDeproxy: Client: %s' % v) + class ServerConnection(asyncore.dispatcher_with_send): @@ -640,17 +687,24 @@ def handle_accept(self): self.connections.append(handler) assert len(self.connections) <= self.conns_n, \ ('Too lot connections, expect %d, got %d' - & (self.conns_n, len(self.connections))) + % (self.conns_n, len(self.connections))) + + def handle_read_event(self): + asyncore.dispatcher.handle_read_event(self) def active_conns_n(self): return len(self.connections) def handle_error(self): _, v, _ = sys.exc_info() - raise Exception('\tDeproxy: Server %s:%d: %s' % (self.ip, self.port, v)) + if type(v) == AssertionError: + raise v + else: + raise Exception('\tDeproxy: Server %s:%d: %s' % \ + (self.ip, self.port, type(v))) def handle_close(self): - self.stop() + self.close() #------------------------------------------------------------------------------- @@ -732,7 +786,7 @@ def run(self): for self.current_chain in self.message_chains: self.recieved_chain = MessageChain.empty() self.client.clear() - self.client.set_request(self.current_chain.request) + self.client.set_request(self.current_chain) self.loop() self.check_expectations() diff --git a/tempesta_fw/t/functional/helpers/wrk.py b/tempesta_fw/t/functional/helpers/wrk.py new file mode 100644 index 0000000000..6557094eda --- /dev/null +++ b/tempesta_fw/t/functional/helpers/wrk.py @@ -0,0 +1,45 @@ +""" Wrk script generator """ + +__author__ = 'Tempesta Technologies, Inc.' +__copyright__ = 'Copyright (C) 2017-2018 Tempesta Technologies, Inc.' +__license__ = 'GPL2' + +from . import remote + +class ScriptGenerator(object): + """ Generate lua script """ + request_type = "GET" + uri = "/" + headers = [] + body = "" + config = "" + def __luaencode(self, value): + # TODO: take care about escaping + # if we have tests with special symbols in content + return value + + def set_request_type(self, request_type): + self.request_type = request_type + + def set_uri(self, uri): + self.uri = uri + + def add_header(self, header_name, header_value): + self.headers.append((header_name, header_value)) + + def set_body(self, body): + self.body = body + + def make_config(self): + """ Generate config and write it to file """ + config = "" + config += "wrk.method = \"%s\"\n" % self.request_type + config +="wrk.path = \"%s\"\n" % self.__luaencode(self.uri) + config += "wrk.headers = {\n" + for header in self.headers: + name = self.__luaencode(header[0]) + value = self.__luaencode(header[1]) + config += " [\"%s\"] = \"%s\",\n" % (name, value) + config += "}\n" + config += "wrk.body = \"%s\"\n" % self.__luaencode(self.body) + return config diff --git a/tempesta_fw/t/functional/long_body/__init__.py b/tempesta_fw/t/functional/long_body/__init__.py new file mode 100644 index 0000000000..2e0da75c1a --- /dev/null +++ b/tempesta_fw/t/functional/long_body/__init__.py @@ -0,0 +1,2 @@ +__all__ = ['test_long_request', 'test_long_response', 'body_generator', + 'test_request_wrong_length', 'tester', 'test_response_wrong_length'] diff --git a/tempesta_fw/t/functional/long_body/body_generator.py b/tempesta_fw/t/functional/long_body/body_generator.py new file mode 100644 index 0000000000..0dc7e96d1a --- /dev/null +++ b/tempesta_fw/t/functional/long_body/body_generator.py @@ -0,0 +1,7 @@ +__author__ = 'Tempesta Technologies, Inc.' +__copyright__ = 'Copyright (C) 2017-2018 Tempesta Technologies, Inc.' +__license__ = 'GPL2' + +def generate_body(length): + """ Generate body of specified length """ + return "x" * length diff --git a/tempesta_fw/t/functional/long_body/test_long_request.py b/tempesta_fw/t/functional/long_body/test_long_request.py new file mode 100644 index 0000000000..0f051c4190 --- /dev/null +++ b/tempesta_fw/t/functional/long_body/test_long_request.py @@ -0,0 +1,51 @@ +""" Testing for long body in request """ + +__author__ = 'Tempesta Technologies, Inc.' +__copyright__ = 'Copyright (C) 2017-2018 Tempesta Technologies, Inc.' +__license__ = 'GPL2' + +import unittest +import body_generator + +from testers import stress +from helpers import tf_cfg, control, tempesta, remote, wrk + +class RequestTestBase(stress.StressTest): + """ Test long request """ + config = "cache 0;\n" + script = None + wrk = None + clients = [] + generator = None + + def create_clients_with_body(self, length): + """ Create wrk client with long request body """ + self.generator = wrk.ScriptGenerator() + self.generator.set_body(body_generator.generate_body(length)) + + self.wrk = control.Wrk() + self.wrk.set_script(self.script, content=self.generator.make_config()) + + self.clients = [self.wrk] + +class RequestTest1k(RequestTestBase): + """ Test long request """ + script = "request_1k" + + def create_clients(self): + self.create_clients_with_body(1024) + + def test(self): + """ Test for 1kbyte body """ + self.generic_test_routine(self.config) + +class RequestTest1M(RequestTestBase): + """ Test long request """ + script = "request_1M" + + def create_clients(self): + self.create_clients_with_body(1024**2) + + def test(self): + """ Test for 1Mbyte body """ + self.generic_test_routine(self.config) diff --git a/tempesta_fw/t/functional/long_body/test_long_response.py b/tempesta_fw/t/functional/long_body/test_long_response.py new file mode 100644 index 0000000000..a77f16462a --- /dev/null +++ b/tempesta_fw/t/functional/long_body/test_long_response.py @@ -0,0 +1,65 @@ +""" Testing for long body in response """ + +__author__ = 'Tempesta Technologies, Inc.' +__copyright__ = 'Copyright (C) 2017-2018 Tempesta Technologies, Inc.' +__license__ = 'GPL2' + +import os +import body_generator + +from testers import stress +from helpers import tf_cfg, control, tempesta, remote + +class ResponseTestBase(stress.StressTest): + """ Test long response """ + config = "cache 0;\n" + filename = "long.bin" + uri = "/" + filename + fullname = "" + + def create_content(self, length): + """ Create content file """ + content = body_generator.generate_body(length) + location = tf_cfg.cfg.get('Server', 'resources') + self.fullname = os.path.join(location, self.filename) + tf_cfg.dbg(3, "Copy %s to %s" % (self.filename, self.fullname)) + remote.server.copy_file(self.fullname, content) + + def remove_content(self): + """ Remove content file """ + if not remote.DEBUG_FILES: + tf_cfg.dbg(3, "Remove %s" % self.fullname) + remote.server.run_cmd("rm %s" % self.fullname) + + def tearDown(self): + super(ResponseTestBase, self).tearDown() + self.remove_content() + + def create_clients(self): + """ Create wrk with specified uri """ + self.clients = [control.Wrk(uri=self.uri)] + + def create_servers_with_body(self, length): + """ Create nginx server with long response body """ + self.create_content(length) + port = tempesta.upstream_port_start_from() + nginx = control.Nginx(listen_port=port) + self.servers = [nginx] + +class ResponseTest1k(ResponseTestBase): + """ 1k test """ + def create_servers(self): + self.create_servers_with_body(1024) + + def test(self): + """ Test for 1kbyte body """ + self.generic_test_routine(self.config) + +class ResponseTest1M(ResponseTestBase): + """ 1M test """ + def create_servers(self): + self.create_servers_with_body(1024**2) + + def test(self): + """ Test for 1Mbyte body """ + self.generic_test_routine(self.config) diff --git a/tempesta_fw/t/functional/long_body/test_request_wrong_length.py b/tempesta_fw/t/functional/long_body/test_request_wrong_length.py new file mode 100644 index 0000000000..6e1ce7c79e --- /dev/null +++ b/tempesta_fw/t/functional/long_body/test_request_wrong_length.py @@ -0,0 +1,191 @@ +""" Testing for missing or wrong body length in request """ + +__author__ = 'Tempesta Technologies, Inc.' +__copyright__ = 'Copyright (C) 2017-2018 Tempesta Technologies, Inc.' +__license__ = 'GPL2' + +import unittest +import body_generator +import os + +from . import tester + +from testers import functional +from helpers import tf_cfg, control, tempesta, remote, deproxy, chains + +def req_body_length(base, length): + """ Generate chain with missing or specified body length """ + for msg in ['request', 'fwd_request']: + field = getattr(base, msg) + field.headers.delete_all('Content-Length') + if length != None: + field.headers.add('Content-Length', '%i' % length) + + base.fwd_request.update() + base.request.build_message() + return base + +def generate_chain(method='GET', expect_400=False, connection=None): + base = chains.base(method=method) + chain = tester.BadLengthMessageChain(request=base.request, + expected_responses=[base.response], + forwarded_request=base.fwd_request, + server_response=base.server_response) + if expect_400: + chain.responses.append(chains.response_400(connection=connection)) + return chain + +class TesterCorrectBodyLength(tester.BadLengthDeproxy): + """ Tester """ + def create_base(self): + base = generate_chain(method='PUT') + return (base, len(base.request.body)) + + def __init__(self, *args, **kwargs): + tester.BadLengthDeproxy.__init__(self, *args, **kwargs) + base = self.create_base() + self.message_chains = [req_body_length(base[0], base[1])] + self.cookies = [] + +class TesterMissingBodyLength(TesterCorrectBodyLength): + """ Tester """ + def create_base(self): + base = generate_chain(method='PUT', expect_400=True) + return (base, None) + +class TesterSmallBodyLength(TesterCorrectBodyLength): + """ Tester """ + def create_base(self): + base = generate_chain(method='PUT', expect_400=True) + return (base, len(base.request.body) - 15) + +class TesterDuplicateBodyLength(deproxy.Deproxy): + def __init__(self, *args, **kwargs): + deproxy.Deproxy.__init__(self, *args, **kwargs) + base = chains.base(method='PUT') + cl = base.request.headers['Content-Length'] + + base.request.headers.add('Content-Length', cl) + base.request.build_message() + + base.fwd_request = deproxy.Request() + + base.response = chains.response_400(connection='keep-alive') + + self.message_chains = [base] + self.cookies = [] + +class TesterInvalidBodyLength(deproxy.Deproxy): + def __init__(self, *args, **kwargs): + deproxy.Deproxy.__init__(self, *args, **kwargs) + base = chains.base(method='PUT') + base.request.headers['Content-Length'] = 'invalid' + base.request.build_message() + base.response = chains.response_400(connection='keep-alive') + base.fwd_request = deproxy.Request() + self.message_chains = [base] + self.cookies = [] + +class TesterSecondBodyLength(TesterDuplicateBodyLength): + def second_length(self, content_length): + len = int(content_length) + return "%i" % (len - 1) + + def expected_response(self): + return chains.response_400(connection='keep-alive') + + def __init__(self, *args, **kwargs): + deproxy.Deproxy.__init__(self, *args, **kwargs) + base = chains.base(method='PUT') + cl = base.request.headers['Content-Length'] + + duplicate = self.second_length(cl) + base.request.headers.add('Content-Length', duplicate) + base.request.build_message() + + base.response = self.expected_response() + + base.fwd_request = deproxy.Request() + + self.message_chains = [base] + self.cookies = [] + + +class RequestCorrectBodyLength(functional.FunctionalTest): + """ Wrong body length """ + config = 'cache 0;\nblock_action error reply;\nblock_action attack reply;\n' + + def create_client(self): + self.client = tester.ClientMultipleResponses() + + def create_tester(self): + self.tester = TesterCorrectBodyLength(self.client, self.servers) + + def test(self): + """ Test """ + self.generic_test_routine(self.config, []) + +class RequestMissingBodyLength(RequestCorrectBodyLength): + """ Wrong body length """ + + def create_tester(self): + self.tester = TesterMissingBodyLength(self.client, self.servers) + + def assert_tempesta(self): + msg = 'Tempesta have errors in processing HTTP %s.' + self.assertEqual(self.tempesta.stats.cl_msg_parsing_errors, 1, + msg=(msg % 'requests')) + self.assertEqual(self.tempesta.stats.srv_msg_parsing_errors, 0, + msg=(msg % 'responses')) + self.assertEqual(self.tempesta.stats.cl_msg_other_errors, 0, + msg=(msg % 'requests')) + self.assertEqual(self.tempesta.stats.srv_msg_other_errors, 0, + msg=(msg % 'responses')) + +class RequestSmallBodyLength(RequestMissingBodyLength): + """ Wrong body length """ + def create_tester(self): + self.tester = TesterSmallBodyLength(self.client, self.servers) + +class RequestDuplicateBodyLength(functional.FunctionalTest): + config = 'cache 0;\nblock_action error reply;\nblock_action attack reply;\n' + + def create_client(self): + self.client = deproxy.Client() + + def create_tester(self): + self.tester = TesterDuplicateBodyLength(self.client, self.servers) + + def test(self): + """ Test """ + self.generic_test_routine(self.config, []) + + def assert_tempesta(self): + msg = 'Tempesta have errors in processing HTTP %s.' + self.assertEqual(self.tempesta.stats.cl_msg_parsing_errors, 1, + msg=(msg % 'requests')) + self.assertEqual(self.tempesta.stats.srv_msg_parsing_errors, 0, + msg=(msg % 'responses')) + self.assertEqual(self.tempesta.stats.cl_msg_other_errors, 0, + msg=(msg % 'requests')) + self.assertEqual(self.tempesta.stats.srv_msg_other_errors, 0, + msg=(msg % 'responses')) + +class RequestSecondBodyLength(RequestDuplicateBodyLength): + def create_tester(self): + self.tester = TesterSecondBodyLength(self.client, self.servers) + + def assert_tempesta(self): + msg = 'Tempesta have errors in processing HTTP %s.' + self.assertEqual(self.tempesta.stats.cl_msg_parsing_errors, 1, + msg=(msg % 'requests')) + self.assertEqual(self.tempesta.stats.srv_msg_parsing_errors, 0, + msg=(msg % 'responses')) + self.assertEqual(self.tempesta.stats.cl_msg_other_errors, 0, + msg=(msg % 'requests')) + self.assertEqual(self.tempesta.stats.srv_msg_other_errors, 0, + msg=(msg % 'responses')) + +class RequestInvalidBodyLength(RequestSecondBodyLength): + def create_tester(self): + self.tester = TesterInvalidBodyLength(self.client, self.servers) diff --git a/tempesta_fw/t/functional/long_body/test_response_wrong_length.py b/tempesta_fw/t/functional/long_body/test_response_wrong_length.py new file mode 100644 index 0000000000..f7ff1e79e6 --- /dev/null +++ b/tempesta_fw/t/functional/long_body/test_response_wrong_length.py @@ -0,0 +1,301 @@ +""" Testing for missing or wrong body length in response """ + +__author__ = 'Tempesta Technologies, Inc.' +__copyright__ = 'Copyright (C) 2017-2018 Tempesta Technologies, Inc.' +__license__ = 'GPL2' + +import unittest +import body_generator +import os + +from . import tester + +from testers import functional +from helpers import tf_cfg, control, tempesta, remote, deproxy, chains + +def resp_body_length(base, length): + """ Generate chain with missing or specified body length """ + for msg in ['response', 'server_response']: + field = getattr(base, msg) + field.headers.delete_all('Content-Length') + if length != None: + actual_len = len(field.body) + if msg == 'response' and actual_len < length: + field.headers.add('Content-Length', '%i' % actual_len) + else: + field.headers.add('Content-Length', '%i' % length) + + base.response.update() + base.server_response.build_message() + return base + +def generate_chain_200(method='GET', response_body=""): + base = chains.base(method=method) + base.response.status = "200" + base.response.body = response_body + base.response.headers['Content-Length'] = len(response_body) + base.server_response.status = "200" + base.server_response.body = response_body + base.server_response.headers['Content-Length'] = len(response_body) + base.response.update() + base.server_response.update() + return base + +def generate_chain_204(method='GET'): + base = chains.base(method=method) + base.response.status = "204" # it's default, but for explicity + base.response.body = "" + base.server_response.status = "204" + base.server_response.body = "" + return base + +class InvalidResponseServer(deproxy.Server): + + def __stop_server(self): + deproxy.Server.__stop_server(self) + assert len(self.connections) <= self.conns_n, \ + ('Too lot connections, expect %d, got %d' + % (self.conns_n, len(self.connections))) + + def handle_accept(self): + pair = self.accept() + if pair is not None: + sock, _ = pair + handler = deproxy.ServerConnection(self.tester, server=self, + sock=sock, + keep_alive=self.keep_alive) + self.connections.append(handler) + +class TesterCorrectEmptyBodyLength(deproxy.Deproxy): + """ Tester """ + def create_base(self): + base = generate_chain_200(method='GET') + return (base, len(base.response.body)) + + def recieved_response(self, response): + """Client received response for its request.""" + self.recieved_chain.response = response + + def __init__(self, *args, **kwargs): + deproxy.Deproxy.__init__(self, *args, **kwargs) + base = self.create_base() + self.message_chains = [resp_body_length(base[0], base[1])] + self.cookies = [] + +class TesterCorrectBodyLength(deproxy.Deproxy): + """ Tester """ + def create_base(self): + base = generate_chain_200(method='GET', response_body="abcd") + return (base, len(base.response.body)) + + def __init__(self, *args, **kwargs): + deproxy.Deproxy.__init__(self, *args, **kwargs) + base = self.create_base() + self.message_chains = [resp_body_length(base[0], base[1])] + self.cookies = [] + +class TesterMissingEmptyBodyLength(deproxy.Deproxy): + """ Tester """ + reply_body = "" + + def __init__(self, *args, **kwargs): + deproxy.Deproxy.__init__(self, *args, **kwargs) + chain = generate_chain_200(method='GET', response_body=self.reply_body) + chain.server_response.headers.delete_all('Content-Length') + chain.server_response.update() + self.message_chains = [chain] + self.cookies = [] + +class TesterMissingBodyLength(TesterMissingEmptyBodyLength): + """ Tester """ + reply_body = "abcdefgh" + +class TesterSmallBodyLength(TesterCorrectBodyLength): + """ Tester """ + def create_base(self): + base = generate_chain_200(method='GET', response_body="abcdefgh") + return (base, len(base.response.body) - 1) + +class TesterForbiddenZeroBodyLength(deproxy.Deproxy): + """ Tester """ + def __init__(self, *args, **kwargs): + deproxy.Deproxy.__init__(self, *args, **kwargs) + base = self.create_base() + base[0].server_response.headers.delete_all('Content-Length') + base[0].server_response.headers.add('Content-Length', "%i" % base[1]) + base[0].server_response.build_message() + + base[0].response = chains.make_502_expected() + + self.message_chains = [base[0]] + self.cookies = [] + + def create_base(self): + base = generate_chain_204(method='GET') + return (base, 0) + +class TesterForbiddenPositiveBodyLength(TesterForbiddenZeroBodyLength): + """ Tester """ + def create_base(self): + base = generate_chain_204(method='GET') + return (base, 1) + +class TesterDuplicateBodyLength(deproxy.Deproxy): + """ Tester """ + def __init__(self, *args, **kwargs): + deproxy.Deproxy.__init__(self, *args, **kwargs) + base = self.create_base() + cl = base[0].server_response.headers['Content-Length'] + base[0].server_response.headers.add('Content-Length', cl) + base[0].server_response.build_message() + + base[0].response = chains.make_502_expected() + + self.message_chains = [base[0]] + self.cookies = [] + + def create_base(self): + base = generate_chain_204(method='GET') + return (base, 0) + +class TesterSecondBodyLength(deproxy.Deproxy): + """ Tester """ + def __init__(self, *args, **kwargs): + deproxy.Deproxy.__init__(self, *args, **kwargs) + base = self.create_base() + cl = base[0].server_response.headers['Content-Length'] + length = int(cl) + base[0].server_response.headers.add('Content-Length', + "%i" % (length - 1)) + base[0].server_response.build_message() + + base[0].response = chains.make_502_expected() + + self.message_chains = [base[0]] + self.cookies = [] + + def create_base(self): + base = generate_chain_204(method='GET') + return (base, 0) + +class TesterInvalidBodyLength(deproxy.Deproxy): + """ Tester """ + def __init__(self, *args, **kwargs): + deproxy.Deproxy.__init__(self, *args, **kwargs) + base = self.create_base() + base[0].server_response.headers['Content-Length'] = "invalid" + base[0].server_response.build_message() + + base[0].response = chains.make_502_expected() + + self.message_chains = [base[0]] + self.cookies = [] + + def create_base(self): + base = generate_chain_204(method='GET') + return (base, 0) + +class ResponseCorrectEmptyBodyLength(functional.FunctionalTest): + """ Correct body length """ + config = 'cache 0;\nblock_action error reply;\nblock_action attack reply;\n' + + def create_servers(self): + port = tempesta.upstream_port_start_from() + self.servers = [InvalidResponseServer(port=port)] + + def create_client(self): + self.client = deproxy.Client() + + def create_tester(self): + self.tester = TesterCorrectEmptyBodyLength(self.client, self.servers) + + def test(self): + """ Test """ + self.generic_test_routine(self.config, []) + +class ResponseCorrectBodyLength(ResponseCorrectEmptyBodyLength): + """ Correct body length """ + + def create_tester(self): + self.tester = TesterCorrectBodyLength(self.client, self.servers) + +class ResponseMissingEmptyBodyLength(ResponseCorrectEmptyBodyLength): + """ Missing body length """ + + def create_servers(self): + port = tempesta.upstream_port_start_from() + self.servers = [deproxy.Server(port=port, keep_alive=1)] + + def create_tester(self): + self.tester = TesterMissingEmptyBodyLength(self.client, self.servers) + +class ResponseMissingBodyLength(ResponseMissingEmptyBodyLength): + """ Missing body length """ + + def create_tester(self): + self.tester = TesterMissingBodyLength(self.client, self.servers) + +class ResponseSmallBodyLength(ResponseCorrectEmptyBodyLength): + """ Small body length """ + + def create_tester(self): + self.tester = TesterSmallBodyLength(self.client, self.servers) + + def assert_tempesta(self): + msg = 'Tempesta have errors in processing HTTP %s.' + self.assertEqual(self.tempesta.stats.cl_msg_parsing_errors, 0, + msg=(msg % 'requests')) + self.assertEqual(self.tempesta.stats.srv_msg_parsing_errors, 0, + msg=(msg % 'responses')) + self.assertEqual(self.tempesta.stats.cl_msg_other_errors, 0, + msg=(msg % 'requests')) + self.assertEqual(self.tempesta.stats.srv_msg_other_errors, 1, + msg=(msg % 'responses')) + +class ResponseForbiddenZeroBodyLength(ResponseCorrectEmptyBodyLength): + """ Forbidden body length """ + + def create_tester(self): + self.tester = TesterForbiddenZeroBodyLength(self.client, self.servers) + + def assert_tempesta(self): + msg = 'Tempesta have errors in processing HTTP %s.' + self.assertEqual(self.tempesta.stats.cl_msg_parsing_errors, 0, + msg=(msg % 'requests')) + self.assertEqual(self.tempesta.stats.srv_msg_parsing_errors, 1, + msg=(msg % 'responses')) + self.assertEqual(self.tempesta.stats.cl_msg_other_errors, 0, + msg=(msg % 'requests')) + self.assertEqual(self.tempesta.stats.srv_msg_other_errors, 0, + msg=(msg % 'responses')) + + +class ResponseForbiddenPositiveBodyLength(ResponseForbiddenZeroBodyLength): + """ Forbidden body length """ + + def create_tester(self): + self.tester = TesterForbiddenPositiveBodyLength(self.client, + self.servers) + +class ResponseDuplicateBodyLength(ResponseCorrectEmptyBodyLength): + def create_tester(self): + self.tester = TesterDuplicateBodyLength(self.client, self.servers) + + def assert_tempesta(self): + msg = 'Tempesta have errors in processing HTTP %s.' + self.assertEqual(self.tempesta.stats.cl_msg_parsing_errors, 0, + msg=(msg % 'requests')) + self.assertEqual(self.tempesta.stats.srv_msg_parsing_errors, 1, + msg=(msg % 'responses')) + self.assertEqual(self.tempesta.stats.cl_msg_other_errors, 0, + msg=(msg % 'requests')) + self.assertEqual(self.tempesta.stats.srv_msg_other_errors, 0, + msg=(msg % 'responses')) + +class ResponseSecondBodyLength(ResponseDuplicateBodyLength): + def create_tester(self): + self.tester = TesterSecondBodyLength(self.client, self.servers) + +class ResponseInvalidBodyLength(ResponseDuplicateBodyLength): + def create_tester(self): + self.tester = TesterInvalidBodyLength(self.client, self.servers) diff --git a/tempesta_fw/t/functional/long_body/tester.py b/tempesta_fw/t/functional/long_body/tester.py new file mode 100644 index 0000000000..fb74138b48 --- /dev/null +++ b/tempesta_fw/t/functional/long_body/tester.py @@ -0,0 +1,88 @@ +__author__ = 'Tempesta Technologies, Inc.' +__copyright__ = 'Copyright (C) 2017-2018 Tempesta Technologies, Inc.' +__license__ = 'GPL2' + +import asyncore +from helpers import tf_cfg, deproxy + +class ClientMultipleResponses(deproxy.Client): + """ Client with support of parsing multiple responses """ + method = "INVALID" + request_buffer = "" + + def set_request(self, request_chain): + if request_chain != None: + self.method = request_chain.method + self.request_buffer = request_chain.request.msg + + def handle_read(self): + self.response_buffer += self.recv(deproxy.MAX_MESSAGE_SIZE) + if not self.response_buffer: + return + tf_cfg.dbg(4, '\tDeproxy: Client: Receive response from Tempesta.') + tf_cfg.dbg(5, self.response_buffer) + + method = self.method + while len(self.response_buffer) > 0: + try: + response = deproxy.Response(self.response_buffer, method=method) + self.response_buffer = self.response_buffer[len(response.msg):] + method = "GET" + except deproxy.ParseError: + tf_cfg.dbg(4, ('Deproxy: Client: Can\'t parse message\n' + '<<<<<\n%s>>>>>' + % self.response_buffer)) + raise + if self.tester: + self.tester.recieved_response(response) + + self.response_buffer = '' + raise asyncore.ExitNow + +class BadLengthMessageChain(deproxy.MessageChain): + def __init__(self, request, expected_responses, forwarded_request=None, + server_response=None): + deproxy.MessageChain.__init__(self, request=request, + forwarded_request=forwarded_request, + server_response=server_response, + expected_response = None) + self.responses = expected_responses + self.method = request.method + + @staticmethod + def empty(): + return BadLengthMessageChain(deproxy.Request(), []) + +class BadLengthDeproxy(deproxy.Deproxy): + """ Support of invalid length """ + def __compare_messages(self, expected, recieved, message): + expected.set_expected(expected_time_delta=self.timeout) + assert expected == recieved, \ + ("Received message (%s) does not suit expected one!\n\n" + "\tReceieved:\n<<<<<|\n%s|>>>>>\n" + "\tExpected:\n<<<<<|\n%s|>>>>>\n" + % (message, recieved.msg, expected.msg)) + + def run(self): + for self.current_chain in self.message_chains: + self.recieved_chain = BadLengthMessageChain.empty() + self.client.clear() + self.client.set_request(self.current_chain) + self.loop() + self.check_expectations() + + def check_expectations(self): + self.__compare_messages(self.current_chain.fwd_request, + self.recieved_chain.fwd_request, 'fwd_request') + nexpected = len(self.current_chain.responses) + nrecieved = len(self.recieved_chain.responses) + assert nexpected == nrecieved, \ + ("Expected %i responses, but recieved %i" % (nexpected, nrecieved)) + for i in range(nexpected): + expected = self.current_chain.responses[i] + recieved = self.recieved_chain.responses[i] + self.__compare_messages(expected, recieved, "response[%i]" % i) + + def recieved_response(self, response): + """Client received response for its request.""" + self.recieved_chain.responses.append(response) diff --git a/tempesta_fw/t/functional/msg_sequence/test_pairing.py b/tempesta_fw/t/functional/msg_sequence/test_pairing.py index 5649e68a98..9eda7c41c3 100644 --- a/tempesta_fw/t/functional/msg_sequence/test_pairing.py +++ b/tempesta_fw/t/functional/msg_sequence/test_pairing.py @@ -66,7 +66,7 @@ def send_reqs(self, req_n): for i in range(b_req, e_req): self.client.clear() - self.client.set_request(self.message_chains[i].request) + self.client.set_request(self.message_chains[i]) while self.client.request_buffer: self.loop(timeout=0.1) diff --git a/tempesta_fw/t/functional/regression/test_invalid.py b/tempesta_fw/t/functional/regression/test_invalid.py index b6f6058a2f..b31ce70fc7 100644 --- a/tempesta_fw/t/functional/regression/test_invalid.py +++ b/tempesta_fw/t/functional/regression/test_invalid.py @@ -3,29 +3,57 @@ from __future__ import print_function import unittest from testers import functional -from helpers import chains, deproxy +from helpers import chains, deproxy, tempesta +from long_body import test_response_wrong_length + __author__ = 'Tempesta Technologies, Inc.' __copyright__ = 'Copyright (C) 2017 Tempesta Technologies, Inc.' __license__ = 'GPL2' +class TesterInvalidResponse(deproxy.Deproxy): + + def recieved_response(self, response): + """Client received response for its request.""" + self.recieved_chain.response = response + class TestInvalidResponse(functional.FunctionalTest): config = ( 'cache 0;\n' ) - @unittest.expectedFailure + def assert_tempesta(self): + """ Assert that tempesta had no errors during test. """ + msg = 'Tempesta have errors in processing HTTP %s.' + self.assertEqual(self.tempesta.stats.cl_msg_parsing_errors, 0, + msg=(msg % 'requests')) + self.assertEqual(self.tempesta.stats.srv_msg_parsing_errors, 1, + msg=(msg % 'responses')) + if not self.tfw_clnt_msg_otherr: + self.assertEqual(self.tempesta.stats.cl_msg_other_errors, 0, + msg=(msg % 'requests')) + self.assertEqual(self.tempesta.stats.srv_msg_other_errors, 0, + msg=(msg % 'responses')) + + + def create_tester(self): + self.tester = TesterInvalidResponse(self.client, self.servers) + + def create_servers(self): + port = tempesta.upstream_port_start_from() + srv = test_response_wrong_length.InvalidResponseServer(port=port) + self.servers = [srv] + def test_204_with_body(self): chain = chains.proxy() - for msg in (chain.server_response, chain.response): - msg.status = '204' - msg.update() + chain.server_response.status = '204' + chain.server_response.build_message() + chain.response = chains.make_502_expected() self.generic_test_routine(self.config, [chain]) - @unittest.expectedFailure def test_no_crlf_before_body(self): chain = chains.proxy() chain.server_response.msg = chain.server_response.msg.replace('\r\n\r\n', '\r\n', 1) - chain.response = deproxy.Response() + chain.response = chains.make_502_expected() self.generic_test_routine(self.config, [chain]) diff --git a/tempesta_fw/t/functional/regression/test_shutdown.py b/tempesta_fw/t/functional/regression/test_shutdown.py index e050d9c66d..2c127d8b26 100644 --- a/tempesta_fw/t/functional/regression/test_shutdown.py +++ b/tempesta_fw/t/functional/regression/test_shutdown.py @@ -100,7 +100,7 @@ def run(self): self.recieved_chain = deproxy.MessageChain.empty() for client in self.clients: client.clear() - client.set_request(self.current_chain.request) + client.set_request(self.current_chain) self.loop() def close_all(self): diff --git a/tempesta_fw/t/functional/sched/test_http.py b/tempesta_fw/t/functional/sched/test_http.py index d377bf0589..58949aae1a 100644 --- a/tempesta_fw/t/functional/sched/test_http.py +++ b/tempesta_fw/t/functional/sched/test_http.py @@ -233,7 +233,7 @@ def configure(self, chain_n): self.recieved_chain = deproxy.MessageChain.empty() self.client.clear() - self.client.set_request(self.current_chain.request) + self.client.set_request(self.current_chain) def recieved_response(self, response): # A lot of clients running, dont raise asyncore.ExitNow directly diff --git a/tempesta_fw/t/functional/testers/functional.py b/tempesta_fw/t/functional/testers/functional.py index d27b7235c4..7079a9f09e 100644 --- a/tempesta_fw/t/functional/testers/functional.py +++ b/tempesta_fw/t/functional/testers/functional.py @@ -3,6 +3,7 @@ import copy import asyncore from helpers import tf_cfg, control, tempesta, deproxy, stateful +from helpers.deproxy import ParseError __author__ = 'Tempesta Technologies, Inc.' __copyright__ = 'Copyright (C) 2017 Tempesta Technologies, Inc.' @@ -146,7 +147,10 @@ def generic_test_routine(self, tempesta_defconfig, message_chains): self.tester.start() tf_cfg.dbg(3, "\tStarting completed") - self.tester.run() + try: + self.tester.run() + except ParseError as err: + self.assertTrue(False, msg=err) self.tempesta.get_stats() self.assert_tempesta()