diff --git a/tests/performance/run_server_performance_test.py b/tests/performance/run_server_performance_test.py new file mode 100644 index 0000000000..2840dfd404 --- /dev/null +++ b/tests/performance/run_server_performance_test.py @@ -0,0 +1,279 @@ +import argparse +import csv +import datetime +import random +import subprocess +import threading +import time +from collections import defaultdict + + +VERBOSE = False + + +def timeit(func): + """ + This decorator makes the applied function return its duration. The original + return value gets lost. + """ + def func_wrapper(*args, **kwargs): + before = datetime.datetime.now() + func(*args, **kwargs) + after = datetime.datetime.now() + return (after - before).total_seconds() + + return func_wrapper + + +def print_process_output(message, stdout, stderr): + global VERBOSE + + if not VERBOSE: + return + + print(message) + print('-' * 20 + 'stdout' + '-' * 20) + print(stdout) + print('-' * 20 + 'stderr' + '-' * 20) + print(stderr) + print('-' * (40 + len('stdout'))) + + +def parse_arguments(): + parser = argparse.ArgumentParser( + description='Performance tester for CodeChecker storage.', + epilog='This test simulates some user actions which are performed on ' + 'a CodeChecker server. The test instantiates the given number ' + 'of users. These users perform a run storage, some queries and ' + 'run deletion. The duration of all tasks is measured. These ' + 'durations are written to the output file at the end of the ' + 'test in CSV format. The tasks are performed for all report ' + 'directories by all users.') + + parser.add_argument('input', + type=str, + metavar='file/folder', + nargs='+', + default='~/.codechecker/reports', + help="The analysis result files and/or folders.") + parser.add_argument('--url', + type=str, + metavar='PRODUCT_URL', + dest='product_url', + default='localhost:8001/Default', + required=True, + help="The URL of the product to store the results " + "for, in the format of host:port/ProductName.") + parser.add_argument('-o', '--output', + type=str, + required=True, + help="Output file name for printing statistics.") + parser.add_argument('-u', '--users', + type=int, + default=1, + help="Number of users") + parser.add_argument('-v', '--verbose', + action='store_true', + help="Print the output of CodeChecker commands.") + + return parser.parse_args() + + +class StatManager: + """ + This class stores the statistics of the single user events and prints them + in CSV format. To produce a nice output the users should do the same tasks + in the same order, e.g. they should all store, query and delete a run + in this order. In the output table a row belongs to each user. The columns + are the durations of the accomplished tasks. + """ + + def __init__(self): + # In this dictionary user ID is mapped to a list of key-value + # pairs: the key is a process name the value is its duration. + self._stats = defaultdict(list) + + def add_duration(self, user_id, task_name, duration): + """ + Add the duration of an event to the statistics. + """ + self._stats[user_id].append((task_name, duration)) + + def print_stats(self, file_name): + if not self._stats: + return + + with open(file_name, 'w') as f: + writer = csv.writer(f) + + _, durations = self._stats.iteritems().next() + header = ['User'] + map(lambda x: x[0], durations) + + writer.writerow(header) + + for user_id, durations in self._stats.iteritems(): + writer.writerow([user_id] + map(lambda x: x[1], durations)) + + +class UserSimulator: + """ + This class simulates a user who performs actions one after the other. The + durations of the single actions are stored in the statistics. + """ + + _counter = 0 + + def __init__(self, stat): + UserSimulator._counter += 1 + self._id = UserSimulator._counter + self._actions = list() + self._stat = stat + + def get_id(self): + return self._id + + def add_action(self, name, func, args): + """ + This function adds a user action to be played later. + name -- The name of the action to identify it in the statistics output. + func -- A function object on which @timeit decorator is applied. + args -- A tuple of function arguments to be passed to func. + """ + self._actions.append((name, func, args)) + + def play(self): + for name, func, args in self._actions: + self._user_random_sleep() + duration = func(*args) + self._stat.add_duration(self._id, name, duration) + + def _user_random_sleep(self): + sec = random.randint(5, 10) + print("User {} is sleeping {} seconds".format(self._id, sec)) + time.sleep(sec) + + +@timeit +def store_report_dir(report_dir, run_name, server_url): + print("Storage of {} is started ({})".format(run_name, report_dir)) + + store_process = subprocess.Popen([ + 'CodeChecker', 'store', + '--url', server_url, + '--name', run_name, + report_dir], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + print_process_output("Output of storage", + *store_process.communicate()) + + print("Storage of {} is done".format(run_name)) + + +@timeit +def local_compare(report_dir, run_name, server_url): + print("Local compare of {} is started ({})".format(run_name, report_dir)) + + compare_process = subprocess.Popen([ + 'CodeChecker', 'cmd', 'diff', + '--url', server_url, + '-b', run_name, + '-n', report_dir, + '--unresolved'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + print_process_output("Output of local compare", + *compare_process.communicate()) + + print("Local compare of {} is done".format(run_name)) + + +@timeit +def get_reports(run_name, server_url): + print("Getting report list for {} is started".format(run_name)) + + report_process = subprocess.Popen([ + 'CodeChecker', 'cmd', 'results', + '--url', server_url, + run_name], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + print_process_output("Output of result list", + *report_process.communicate()) + + print("Getting report list for {} is done".format(run_name)) + + +@timeit +def delete_run(run_name, server_url): + print("Deleting run {} is started".format(run_name)) + + delete_process = subprocess.Popen([ + 'CodeChecker', 'cmd', 'del', + '--url', server_url, + '-n', run_name], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + print_process_output("Output of run deletion", + *delete_process.communicate()) + + print("Deleting run {} is done".format(run_name)) + + +def simulate_user(report_dirs, server_url, stat): + user = UserSimulator(stat) + run_name = 'performance_test_' + str(user.get_id()) + + for report_dir in report_dirs: + user.add_action( + 'Storage', + store_report_dir, + (report_dir, run_name, server_url)) + + user.add_action( + 'Comparison', + local_compare, + (report_dir, run_name, server_url)) + + user.add_action( + 'Reports', + get_reports, + (run_name, server_url)) + + user.add_action( + 'Delete', + delete_run, + (run_name, server_url)) + + user.play() + + +def main(): + global VERBOSE + + args = parse_arguments() + + VERBOSE = args.verbose + + stat = StatManager() + + threads = [threading.Thread( + target=simulate_user, + args=(args.input, args.product_url, stat)) + for _ in range(args.users)] + + for t in threads: + t.start() + + for t in threads: + t.join() + + stat.print_stats(args.output) + + +if __name__ == '__main__': + main()