From b139ca6ec4e7752200aa61566c9611a12ebdd785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Csord=C3=A1s?= Date: Fri, 8 Jun 2018 11:59:04 +0200 Subject: [PATCH] Allow comparison of two report directories * It is possible to compare two report directory. * Handle cases when basename is a report directory and newname is a remote run. * Filter the results of local comparison. * Test cases for local comparison. --- docs/user_guide.md | 17 +- libcodechecker/cmd/cmd_line_client.py | 442 ++++++++++++++++++-------- libcodechecker/libhandlers/cmd.py | 26 +- libcodechecker/report.py | 4 + tests/functional/diff/__init__.py | 12 +- tests/functional/diff/test_diff.py | 231 +++++++++----- 6 files changed, 502 insertions(+), 230 deletions(-) diff --git a/docs/user_guide.md b/docs/user_guide.md index a35f749ec9..b940f772f3 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -1400,13 +1400,20 @@ Results can be filtered by using separate filter options of `results`, `diff`, ~~~~~~~~~~~~~~~~~~~~~ filter arguments: --review-status [REVIEW_STATUS [REVIEW_STATUS ...]] - Filter results by review statuses. + Filter results by review statuses. This can be used + only if basename or newname is a run name (on the + remote server). (default: ['unreviewed', 'confirmed']) --detection-status [DETECTION_STATUS [DETECTION_STATUS ...]] - Filter results by detection statuses. + Filter results by detection statuses. This can be used + only if basename or newname is a run name (on the + remote server). (default: ['new', 'reopened', + 'unresolved']) --severity [SEVERITY [SEVERITY ...]] Filter results by severities. --tag [TAG [TAG ...]] - Filter results by version tag names. + Filter results by version tag names. This can be used + only if basename or newname is a run name (on the + remote server). --file [FILE_PATH [FILE_PATH ...]] Filter results by file path. The file path can contain multiple * quantifiers which matches any number of @@ -1422,7 +1429,9 @@ filter arguments: can contain multiple * quantifiers which matches any number of characters (zero or more). --component [COMPONENT [COMPONENT ...]] - Filter results by source components. + Filter results by source components. This can be used + only if basename or newname is a run name (on the + remote server). -s, --suppressed DEPRECATED. Use the '--filter' option to get false positive (suppressed) results. Show only suppressed results instead of only unsuppressed ones. diff --git a/libcodechecker/cmd/cmd_line_client.py b/libcodechecker/cmd/cmd_line_client.py index 76ffb4d9b0..7212af08f0 100644 --- a/libcodechecker/cmd/cmd_line_client.py +++ b/libcodechecker/cmd/cmd_line_client.py @@ -120,12 +120,11 @@ def validate_filter_values(user_values, valid_values, value_type): return True -def add_filter_conditions(client, report_filter, args): +def check_filter_values(args): """ - This function fills some attributes of the given report filter based on - the arguments which is provided in the command line. + Check if filter values are valid values. Returns values which are checked + or exit from the interpreter. """ - severities = checkers = file_path = dt_statuses = rw_statuses = None filter_str = args.filter if 'filter' in args else None @@ -161,31 +160,52 @@ def add_filter_conditions(client, report_filter, args): values_to_check = [ (severities, ttypes.Severity._NAMES_TO_VALUES, 'severity'), (dt_statuses, ttypes.DetectionStatus._NAMES_TO_VALUES, - 'detection status'), + 'detection status'), (rw_statuses, ttypes.ReviewStatus._NAMES_TO_VALUES, - 'review status')] + 'review status')] if not all(valid for valid in [validate_filter_values(*x) for x in values_to_check]): sys.exit(1) + return severities, checkers, file_path, dt_statuses, rw_statuses + + +def add_filter_conditions(client, report_filter, args): + """ + This function fills some attributes of the given report filter based on + the arguments which is provided in the command line. + """ + + severities, checkers, file_path, dt_statuses, rw_statuses = \ + check_filter_values(args) if severities: report_filter.severity = map( lambda x: ttypes.Severity._NAMES_TO_VALUES[x.upper()], severities) + if dt_statuses: + report_filter.detectionStatus = map( + lambda x: ttypes.DetectionStatus._NAMES_TO_VALUES[x.upper()], + dt_statuses) + + if rw_statuses: + report_filter.reviewStatus = map( + lambda x: ttypes.ReviewStatus._NAMES_TO_VALUES[x.upper()], + rw_statuses) + if checkers: report_filter.checkerName = checkers + if file_path: + report_filter.filepath = file_path + if 'checker_msg' in args: report_filter.checkerMsg = args.checker_msg if 'component' in args: report_filter.componentNames = args.component - if file_path: - report_filter.filepath = file_path - if 'tag' in args: run_history_filter = ttypes.RunHistoryFilter(tagNames=args.tag) run_histories = client.getRunHistory(None, None, None, @@ -193,16 +213,6 @@ def add_filter_conditions(client, report_filter, args): if run_histories: report_filter.runTag = [t.id for t in run_histories] - if dt_statuses: - report_filter.detectionStatus = map( - lambda x: ttypes.DetectionStatus._NAMES_TO_VALUES[x.upper()], - dt_statuses) - - if rw_statuses: - report_filter.reviewStatus = map( - lambda x: ttypes.ReviewStatus._NAMES_TO_VALUES[x.upper()], - rw_statuses) - # --------------------------------------------------------------------------- # Argument handlers for the 'CodeChecker cmd' subcommands. # --------------------------------------------------------------------------- @@ -294,42 +304,43 @@ def handle_diff_results(args): init_logger(args.verbose if 'verbose' in args else None) check_deprecated_arg_usage(args) - context = generic_package_context.get_context() - - def get_diff_results(client, baseids, report_filter, cmp_data): - add_filter_conditions(client, report_filter, args) - - # Do not show resolved bugs in compare mode new. - if cmp_data.diffType == ttypes.DiffType.NEW: - report_filter.detectionStatus = [ - ttypes.DetectionStatus.NEW, - ttypes.DetectionStatus.UNRESOLVED, - ttypes.DetectionStatus.REOPENED] + f_severities, f_checkers, f_file_path, _, _ = check_filter_values(args) - sort_mode = [(ttypes.SortMode( - ttypes.SortType.FILENAME, - ttypes.Order.ASC))] - limit = constants.MAX_QUERY_SIZE - offset = 0 + context = generic_package_context.get_context() - all_results = [] - results = client.getRunResults(baseids, - limit, - offset, - sort_mode, - report_filter, - cmp_data) + def skip_report_dir_result(report): + """ + Returns True if the report should be skipped from the results based on + the given filter set. + """ + if f_severities: + severity_name = context.severity_map.get(report.main['check_name'], + 'UNSPECIFIED') + if severity_name.lower() not in map(str.lower, f_severities): + return True + + if f_checkers: + checker_name = report.main['check_name'] + if not any([re.match(r'^' + c.replace("*", ".*") + '$', + checker_name, re.IGNORECASE) + for c in f_checkers]): + return True + + if f_file_path: + file_path = report.files[int(report.main['location']['file'])] + if not any([re.match(r'^' + f.replace("*", ".*") + '$', + file_path, re.IGNORECASE) + for f in f_file_path]): + return True + + if 'checker_msg' in args: + checker_msg = report.main['description'] + if not any([re.match(r'^' + c.replace("*", ".*") + '$', + checker_msg, re.IGNORECASE) + for c in args.checker_msg]): + return True - while results: - all_results.extend(results) - offset += limit - results = client.getRunResults(baseids, - limit, - offset, - sort_mode, - report_filter, - cmp_data) - return all_results + return False def get_report_dir_results(reportdir): all_reports = [] @@ -350,6 +361,9 @@ def get_report_dir_results(reportdir): LOG.debug(report) continue + if skip_report_dir_result(report): + continue + processed_path_hashes.add(path_hash) report.main['location']['file_name'] = \ files[int(report.main['location']['file'])] @@ -398,19 +412,17 @@ def get_diff_base_results(client, baseids, base_hashes, suppressed_hashes): None) return base_results - def get_diff_report_dir(client, baseids, report_dir, cmp_data): - filtered_reports = [] - report_dir_results = get_report_dir_results(report_dir) - new_hashes = {} + def get_suppressed_reports(reports): + """ + Returns suppressed reports. + """ suppressed_in_code = [] - - for rep in report_dir_results: - bughash = rep.main['issue_hash_content_of_line_in_context'] + for rep in reports: + bughash = rep.report_hash source_file = rep.main['location']['file_name'] bug_line = rep.main['location']['line'] checker_name = rep.main['check_name'] - new_hashes[bughash] = rep sc_handler = SourceCodeCommentHandler(source_file) src_comment_data = sc_handler.filter_source_line_comments( bug_line, @@ -427,30 +439,183 @@ def get_diff_report_dir(client, baseids, report_dir, cmp_data): "for '{0}' checker in '{1}' at line {2}. " "This bug will not be suppressed!".format( checker_name, source_file, bug_line)) + return suppressed_in_code - base_hashes = client.getDiffResultsHash(baseids, - new_hashes.keys(), - cmp_data.diffType) + def get_diff_type(): + """ + Returns Thrift DiffType value by processing the arguments. + """ + if 'new' in args: + return ttypes.DiffType.NEW + elif 'unresolved' in args: + return ttypes.DiffType.UNRESOLVED + elif 'resolved' in args: + return ttypes.DiffType.RESOLVED + else: + return None - if cmp_data.diffType == ttypes.DiffType.NEW or \ - cmp_data.diffType == ttypes.DiffType.UNRESOLVED: - # Shows reports from the report dir which are not present in the - # baseline (NEW reports) or appear in both side (UNRESOLVED + def get_diff_local_dir_remote_run(client, report_dir, run_name): + """ + Compares a local report directory with a remote run. + """ + filtered_reports = [] + report_dir_results = get_report_dir_results( + os.path.abspath(report_dir)) + suppressed_in_code = get_suppressed_reports(report_dir_results) + + diff_type = get_diff_type() + run_ids, _ = process_run_arg(run_name) + local_report_hashes = set([r.report_hash for r in report_dir_results]) + + if diff_type == ttypes.DiffType.NEW: + # Get report hashes which can be found only in the remote runs. + remote_report_hashes = \ + client.getDiffResultsHash(run_ids, + local_report_hashes, + ttypes.DiffType.RESOLVED) + + results = get_diff_base_results(client, run_ids, + remote_report_hashes, + suppressed_in_code) + for result in results: + filtered_reports.append(result) + elif diff_type == ttypes.DiffType.UNRESOLVED: + # Get remote hashes which can be found in the remote run and in the + # local report directory. + remote_report_hashes = \ + client.getDiffResultsHash(run_ids, + local_report_hashes, + ttypes.DiffType.UNRESOLVED) + for result in report_dir_results: + h = result.report_hash + if h in remote_report_hashes and h not in suppressed_in_code: + filtered_reports.append(result) + elif diff_type == ttypes.DiffType.RESOLVED: + # Get remote hashes which can be found in the remote run and in the + # local report directory. + remote_report_hashes = \ + client.getDiffResultsHash(run_ids, + local_report_hashes, + ttypes.DiffType.UNRESOLVED) + for result in report_dir_results: + if result.report_hash not in remote_report_hashes: + filtered_reports.append(result) + + return filtered_reports + + def get_diff_remote_run_local_dir(client, run_name, report_dir): + """ + Compares a remote run with a local report directory. + """ + filtered_reports = [] + report_dir_results = get_report_dir_results( + os.path.abspath(report_dir)) + suppressed_in_code = get_suppressed_reports(report_dir_results) + + diff_type = get_diff_type() + run_ids, _ = process_run_arg(run_name) + local_report_hashes = set([r.report_hash for r in report_dir_results]) + + remote_report_hashes = client.getDiffResultsHash(run_ids, + local_report_hashes, + diff_type) + + if diff_type in [ttypes.DiffType.NEW, ttypes.DiffType.UNRESOLVED]: + # Shows reports from the report dir which are not present in + # the baseline (NEW reports) or appear in both side (UNRESOLVED # reports) and not suppressed in the code. for result in report_dir_results: - h = result.main['issue_hash_content_of_line_in_context'] - if h in base_hashes and h not in suppressed_in_code: + h = result.report_hash + if h in remote_report_hashes and h not in suppressed_in_code: filtered_reports.append(result) - elif cmp_data.diffType == ttypes.DiffType.RESOLVED: - # Show bugs in the baseline (server) which are not present in the - # report dir or suppressed. - results = get_diff_base_results(client, baseids, base_hashes, + elif diff_type == ttypes.DiffType.RESOLVED: + # Show bugs in the baseline (server) which are not present in + # the report dir or suppressed. + results = get_diff_base_results(client, + run_ids, + remote_report_hashes, suppressed_in_code) for result in results: filtered_reports.append(result) return filtered_reports + def get_diff_remote_runs(client, basename, newname): + """ + Compares two remote runs and returns the filtered results. + """ + report_filter = ttypes.ReportFilter() + add_filter_conditions(client, report_filter, args) + + base_ids, base_run_tags = process_run_arg(basename) + report_filter.runTag = base_run_tags + + cmp_data = ttypes.CompareData() + cmp_data.diffType = get_diff_type() + + new_ids, new_run_tags = process_run_arg(newname) + cmp_data.runIds = new_ids + cmp_data.runTag = new_run_tags + + # Do not show resolved bugs in compare mode new. + if cmp_data.diffType == ttypes.DiffType.NEW: + report_filter.detectionStatus = [ + ttypes.DetectionStatus.NEW, + ttypes.DetectionStatus.UNRESOLVED, + ttypes.DetectionStatus.REOPENED] + + sort_mode = [(ttypes.SortMode( + ttypes.SortType.FILENAME, + ttypes.Order.ASC))] + limit = constants.MAX_QUERY_SIZE + offset = 0 + + all_results = [] + results = client.getRunResults(base_ids, + limit, + offset, + sort_mode, + report_filter, + cmp_data) + + while results: + all_results.extend(results) + offset += limit + results = client.getRunResults(base_ids, + limit, + offset, + sort_mode, + report_filter, + cmp_data) + return all_results + + def get_diff_local_dirs(basename, newname): + """ + Compares two report directories and returns the filtered results. + """ + filtered_reports = [] + base_results = get_report_dir_results(os.path.abspath(basename)) + new_results = get_report_dir_results(os.path.abspath(newname)) + + base_hashes = set([res.report_hash for res in base_results]) + new_hashes = set([res.report_hash for res in new_results]) + + diff_type = get_diff_type() + if diff_type == ttypes.DiffType.NEW: + for res in new_results: + if res.report_hash not in base_hashes: + filtered_reports.append(res) + if diff_type == ttypes.DiffType.UNRESOLVED: + for res in new_results: + if res.report_hash in base_hashes: + filtered_reports.append(res) + elif diff_type == ttypes.DiffType.RESOLVED: + for res in base_results: + if res.report_hash not in new_hashes: + filtered_reports.append(res) + + return filtered_reports + def cached_report_file_lookup(file_cache, file_id): """ Get source file data for the given file and caches it in a file cache @@ -633,23 +798,23 @@ def print_reports(client, reports, output_format): ttypes.LinesInFilesRequested(fileId=key, lines=source_lines[key])) - source_line_contents = client.getLinesInSourceFileContents( - lines_in_files_requested, ttypes.Encoding.BASE64) - for report in reports: if isinstance(report, Report): # report is coming from a plist file. bug_line = report.main['location']['line'] bug_col = report.main['location']['col'] - sev = 'unknown' checked_file = report.main['location']['file_name']\ + ':' + str(bug_line) + ":" + str(bug_col) check_name = report.main['check_name'] + sev = context.severity_map.get(check_name, 'UNSPECIFIED') check_msg = report.main['description'] source_line =\ get_line_from_file(report.main['location']['file_name'], bug_line) else: + source_line_contents = client.getLinesInSourceFileContents( + lines_in_files_requested, ttypes.Encoding.BASE64) + # report is of ReportData type coming from CodeChecker server. bug_line = report.line bug_col = report.column @@ -661,11 +826,11 @@ def print_reports(client, reports, output_format): check_name = report.checkerId check_msg = report.checkerMsg rows.append( - (checked_file, check_name, sev, check_msg, source_line)) + (sev, checked_file, check_msg, check_name, source_line)) if output_format == 'plaintext': for row in rows: - print("{0}: {1} [{2}]\n{3}\n".format(row[0], - row[3], row[1], row[4])) + print("[{0}] {1}: {2} [{3}]\n{4}\n".format( + row[0], row[1], row[2], row[3], row[4])) else: print(twodim_to_str(output_format, header, rows)) @@ -680,72 +845,77 @@ def get_run_tag(client, run_ids, tag_name): return run_histories[0] if len(run_histories) else None - client = setup_client(args.product_url) - - report_filter = ttypes.ReportFilter() - - # Process base run names and tags. - run_with_tag = args.basename.split(':') - base_run_name = run_with_tag[0] - - base_runs = get_runs(client, [base_run_name]) - base_ids = map(lambda run: run.runId, base_runs) - - # Set base run tag if it is available. - run_tag_name = run_with_tag[1] if len(run_with_tag) > 1 else None - if run_tag_name: - tag = get_run_tag(client, base_ids, run_tag_name) - report_filter.runTag = [tag.id] if tag else None + def process_run_arg(run_arg_with_tag): + """ + Process the argument and returns run ids a run tag ids. + The argument has the following format: : + """ + run_with_tag = run_arg_with_tag.split(':') + run_name = run_with_tag[0] - if len(base_ids) == 0: - LOG.warning("No run names match the given pattern: " + args.basename) - sys.exit(1) + runs = get_runs(client, [run_name]) + run_ids = map(lambda run: run.runId, runs) - LOG.info("Matching base runs: " + - ', '.join(map(lambda run: run.name, base_runs))) - - cmp_data = ttypes.CompareData() - if 'new' in args: - cmp_data.diffType = ttypes.DiffType.NEW - elif 'unresolved' in args: - cmp_data.diffType = ttypes.DiffType.UNRESOLVED - elif 'resolved' in args: - cmp_data.diffType = ttypes.DiffType.RESOLVED - - results = [] - if os.path.isdir(args.newname): - # If newname is a valid directory we assume that it is a report dir and - # we are in local compare mode. - results = get_diff_report_dir(client, base_ids, - os.path.abspath(args.newname), - cmp_data) - else: - run_with_tag = args.newname.split(':') - new_run_name = run_with_tag[0] + # Set base run tag if it is available. run_tag_name = run_with_tag[1] if len(run_with_tag) > 1 else None + run_tags = None + if run_tag_name: + tag = get_run_tag(client, run_ids, run_tag_name) + run_tags = [tag.id] if tag else None - new_runs = get_runs(client, [new_run_name]) - new_ids = map(lambda run: run.runId, new_runs) - - if len(new_ids) == 0: + if not run_ids: LOG.warning( - "No run names match the given pattern: " + args.newname) + "No run names match the given pattern: " + run_arg_with_tag) sys.exit(1) - LOG.info("Matching new runs: " + - ', '.join(map(lambda run: run.name, new_runs))) - - cmp_data.runIds = new_ids - if run_tag_name: - tag = get_run_tag(client, new_ids, run_tag_name) - cmp_data.runTag = [tag.id] if tag else None + LOG.info("Matching runs: %s", + ', '.join(map(lambda run: run.name, runs))) - results = get_diff_results(client, base_ids, report_filter, cmp_data) + return run_ids, run_tags - if len(results) == 0: - LOG.info("No results.") + def print_diff_results(reports): + """ + Print the results. + """ + if len(reports) == 0: + LOG.info("No results.") + else: + print_reports(client, reports, args.output_format) + + client = None + + # We set up the client if we are not comparing two local report directory. + if not os.path.isdir(args.basename) or not os.path.isdir(args.newname): + client = setup_client(args.product_url) + + if os.path.isdir(args.basename) and os.path.isdir(args.newname): + reports = get_diff_local_dirs(args.basename, args.newname) + print_diff_results(reports) + LOG.info("Compared two local report directories %s and %s", + os.path.abspath(args.basename), + os.path.abspath(args.newname)) + elif os.path.isdir(args.newname): + reports = get_diff_remote_run_local_dir(client, + args.basename, + os.path.abspath(args.newname)) + print_diff_results(reports) + LOG.info("Compared remote run %s and local report directory %s", + args.basename, + os.path.abspath(args.newname)) + elif os.path.isdir(args.basename): + reports = get_diff_local_dir_remote_run(client, + os.path.abspath(args.basename), + args.newname) + print_diff_results(reports) + LOG.info("Compared local report directory %s and remote run %s.", + os.path.abspath(args.basename), + args.newname) else: - print_reports(client, results, args.output_format) + reports = get_diff_remote_runs(client, args.basename, args.newname) + print_diff_results(reports) + LOG.info("Compared two remote runs %s and %s", + args.basename, + args.newname) def handle_list_result_types(args): diff --git a/libcodechecker/libhandlers/cmd.py b/libcodechecker/libhandlers/cmd.py index 695a02bafe..73c0a4c373 100644 --- a/libcodechecker/libhandlers/cmd.py +++ b/libcodechecker/libhandlers/cmd.py @@ -166,7 +166,7 @@ def __add_common_arguments(parser, logger.add_verbose_arguments(common_group) -def __add_filtering_arguments(parser, defaults=None): +def __add_filtering_arguments(parser, defaults=None, diff_mode=False): """ Add some common filtering arguments to the given parser. """ @@ -177,19 +177,26 @@ def init_default(dest): f_group = parser.add_argument_group('filter arguments') + warn_diff_mode = "" + if diff_mode: + warn_diff_mode = " This can be used only if basename or newname is " \ + "a run name (on the remote server)." + f_group.add_argument('--review-status', nargs='*', dest="review_status", metavar='REVIEW_STATUS', default=init_default('review_status'), - help="Filter results by review statuses.") + help="Filter results by review statuses." + + warn_diff_mode) f_group.add_argument('--detection-status', nargs='*', dest="detection_status", metavar='DETECTION_STATUS', default=init_default('detection_status'), - help="Filter results by detection statuses.") + help="Filter results by detection statuses." + + warn_diff_mode) f_group.add_argument('--severity', nargs='*', @@ -203,7 +210,8 @@ def init_default(dest): dest="tag", metavar='TAG', default=init_default('tag'), - help="Filter results by version tag names.") + help="Filter results by version tag names." + + warn_diff_mode) f_group.add_argument('--file', nargs='*', @@ -244,7 +252,8 @@ def init_default(dest): dest="component", metavar='COMPONENT', default=argparse.SUPPRESS, - help="Filter results by source components.") + help="Filter results by source components." + + warn_diff_mode) f_group.add_argument('-s', '--suppressed', default=argparse.SUPPRESS, @@ -306,7 +315,10 @@ def __register_diff(parser): default=argparse.SUPPRESS, help="The 'base' (left) side of the difference: this " "analysis run is used as the initial state in " - "the comparison. The basename can contain * " + "the comparison. The parameter can be a run name " + "(on the remote server) or a local report " + "directory (result of the analyze command). In " + "case of run name the the basename can contain * " "quantifiers which matches any number of " "characters (zero or more). So if you have " "run-a-1, run-a-2 and run-b-1 " @@ -330,7 +342,7 @@ def __register_diff(parser): "run-a-1, run-a-2 and run-b-1 " "then \"run-a*\" selects the first two.") - __add_filtering_arguments(parser, DEFAULT_FILTER_VALUES) + __add_filtering_arguments(parser, DEFAULT_FILTER_VALUES, True) group = parser.add_argument_group("comparison modes") group = group.add_mutually_exclusive_group(required=True) diff --git a/libcodechecker/report.py b/libcodechecker/report.py index 781cc56071..566509598a 100644 --- a/libcodechecker/report.py +++ b/libcodechecker/report.py @@ -196,6 +196,10 @@ def __init__(self, main, bugpath, files): def main(self): return self.__main + @property + def report_hash(self): + return self.__main['issue_hash_content_of_line_in_context'] + @property def bug_path(self): return self.__bug_path diff --git a/tests/functional/diff/__init__.py b/tests/functional/diff/__init__.py index e164382508..891b85c86d 100644 --- a/tests/functional/diff/__init__.py +++ b/tests/functional/diff/__init__.py @@ -100,6 +100,9 @@ def setup_package(): sys.exit(1) print("First analysis of the test project was successful.") + dir_util.copy_tree(codechecker_cfg['reportdir'], + os.path.join(TEST_WORKSPACE, 'reports_baseline')) + ret = project.clean(test_project, test_env) if ret: sys.exit(ret) @@ -124,15 +127,6 @@ def setup_package(): "call_and_message.cpp") insert_suppression(altered_file) - # Run the second analysis results - # into a report directory - ret = codechecker.analyze(codechecker_cfg, - test_project_name_new, - test_project_path_altered) - if ret: - sys.exit(1) - print("CodeChecker analyze of test project was successful.") - test_project_name_update = project_info['name'] + '_' + uuid.uuid4().hex codechecker_cfg['tag'] = 't1' codechecker_cfg['checkers'] = ['-d', 'core.CallAndMessage', diff --git a/tests/functional/diff/test_diff.py b/tests/functional/diff/test_diff.py index 42dce04b10..b5794d817c 100644 --- a/tests/functional/diff/test_diff.py +++ b/tests/functional/diff/test_diff.py @@ -70,6 +70,8 @@ def setUp(self): self._codechecker_cmd = env.codechecker_cmd() self._report_dir = os.path.join(test_workspace, "reports") + self._report_dir_baseline = os.path.join(test_workspace, + "reports_baseline") self._test_config = env.import_test_cfg(test_workspace) self._run_names = env.get_run_names(test_workspace) self._html_reports = os.path.join(test_workspace, "html_reports") @@ -557,94 +559,175 @@ def test_cmd_compare_remote_res_count_new(self): count = len(re.findall(r'\[core\.StackAddressEscape\]', out)) self.assertEqual(count, 3) - def test_local_compare_res_count_resovled(self): + def test_local_cmp_count_resolved(self): """ Count the resolved results with no filter in local compare mode. """ - base_run_name = self._run_names[0] - diff_cmd = [self._codechecker_cmd, "cmd", "diff", - "--resolved", - "--url", self._url, - "-b", base_run_name, - "-n", self._report_dir - ] - print(diff_cmd) - process = subprocess.Popen( - diff_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=self._test_config['codechecker_cfg']['check_env'], - cwd=os.environ['TEST_WORKSPACE']) - out, err = process.communicate() - print(out+err) - - # # 3 disappeared core.StackAddressEscape issues - count = len(re.findall(r'\[core\.StackAddressEscape\]', out)) - self.assertEqual(count, 3) - - def test_local_compare_res_count_unresovled(self): + def test_local_compare(baseline, newcheck): + diff_cmd = [self._codechecker_cmd, "cmd", "diff", + "--resolved", + "--url", self._url, + "-b", baseline, + "-n", newcheck + ] + print(diff_cmd) + process = subprocess.Popen( + diff_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=self._test_config['codechecker_cfg']['check_env'], + cwd=os.environ['TEST_WORKSPACE']) + out, err = process.communicate() + print(out+err) + + # 3 disappeared core.StackAddressEscape issues + count = len(re.findall(r'\[core\.StackAddressEscape\]', out)) + self.assertEqual(count, 3) + + test_local_compare(self._run_names[0], self._report_dir) + test_local_compare(self._report_dir_baseline, self._run_names[1]) + test_local_compare(self._report_dir_baseline, self._report_dir) + + def test_local_cmp_count_unres(self): """ Count the unresolved results with no filter in local compare mode. """ - base_run_name = self._run_names[0] - diff_cmd = [self._codechecker_cmd, "cmd", "diff", - "--unresolved", - "--url", self._url, - "-b", base_run_name, - "-n", self._report_dir - ] - print(diff_cmd) - process = subprocess.Popen( - diff_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=self._test_config['codechecker_cfg']['check_env'], - cwd=os.environ['TEST_WORKSPACE']) - out, err = process.communicate() - print(out+err) - - # # 3 disappeared core.StackAddressEscape issues - count = len(re.findall(r'\[core\.DivideZero\]', out)) - self.assertEqual(count, 10) - count = len(re.findall(r'\[deadcode\.DeadStores\]', out)) - self.assertEqual(count, 6) - count = len(re.findall(r'\[core\.NullDereference\]', out)) - self.assertEqual(count, 4) - count = len(re.findall(r'\[cplusplus\.NewDelete\]', out)) - self.assertEqual(count, 5) - count = len(re.findall(r'\[unix\.Malloc\]', out)) - self.assertEqual(count, 1) - - def test_local_compare_res_count_unresovled_regex(self): + def test_local_compare(baseline, newcheck): + diff_cmd = [self._codechecker_cmd, "cmd", "diff", + "--unresolved", + "--url", self._url, + "-b", baseline, + "-n", newcheck + ] + print(diff_cmd) + process = subprocess.Popen( + diff_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=self._test_config['codechecker_cfg']['check_env'], + cwd=os.environ['TEST_WORKSPACE']) + out, err = process.communicate() + print(out+err) + + # # 3 disappeared core.StackAddressEscape issues + count = len(re.findall(r'\[core\.DivideZero\]', out)) + self.assertEqual(count, 10) + count = len(re.findall(r'\[deadcode\.DeadStores\]', out)) + self.assertEqual(count, 6) + count = len(re.findall(r'\[core\.NullDereference\]', out)) + self.assertEqual(count, 4) + count = len(re.findall(r'\[cplusplus\.NewDelete\]', out)) + self.assertEqual(count, 5) + count = len(re.findall(r'\[unix\.Malloc\]', out)) + self.assertEqual(count, 1) + + test_local_compare(self._run_names[0], self._report_dir) + test_local_compare(self._report_dir_baseline, self._run_names[1]) + test_local_compare(self._report_dir_baseline, self._report_dir) + + def test_local_cmp_count_unres_rgx(self): """ Count the unresolved results with no filter in local compare mode. """ + def test_local_compare(baseline, newcheck): + diff_cmd = [self._codechecker_cmd, "cmd", "diff", + "--unresolved", + "--url", self._url, + "-b", baseline, + "-n", newcheck + ] + print(diff_cmd) + process = subprocess.Popen( + diff_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=self._test_config['codechecker_cfg']['check_env'], + cwd=os.environ['TEST_WORKSPACE']) + out, err = process.communicate() + print(out+err) + + # # 3 disappeared core.StackAddressEscape issues + count = len(re.findall(r'\[core\.DivideZero\]', out)) + self.assertEqual(count, 10) + count = len(re.findall(r'\[deadcode\.DeadStores\]', out)) + self.assertEqual(count, 6) + count = len(re.findall(r'\[core\.NullDereference\]', out)) + self.assertEqual(count, 4) + count = len(re.findall(r'\[cplusplus\.NewDelete\]', out)) + self.assertEqual(count, 5) + count = len(re.findall(r'\[unix\.Malloc\]', out)) + self.assertEqual(count, 1) + base_run_name = self._run_names[0] # Change test_files_blablabla to test_*_blablabla base_run_name = base_run_name.replace('files', '*') - diff_cmd = [self._codechecker_cmd, "cmd", "diff", - "--unresolved", - "--url", self._url, - "-b", base_run_name, - "-n", self._report_dir - ] - print(diff_cmd) - process = subprocess.Popen( - diff_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=self._test_config['codechecker_cfg']['check_env'], - cwd=os.environ['TEST_WORKSPACE']) - out, err = process.communicate() - print(out+err) - - # # 3 disappeared core.StackAddressEscape issues - count = len(re.findall(r'\[core\.DivideZero\]', out)) - self.assertEqual(count, 10) - count = len(re.findall(r'\[deadcode\.DeadStores\]', out)) - self.assertEqual(count, 6) - count = len(re.findall(r'\[core\.NullDereference\]', out)) - self.assertEqual(count, 4) - count = len(re.findall(r'\[cplusplus\.NewDelete\]', out)) - self.assertEqual(count, 5) - count = len(re.findall(r'\[unix\.Malloc\]', out)) - self.assertEqual(count, 1) + test_local_compare(base_run_name, self._report_dir) + test_local_compare(self._report_dir_baseline, self._run_names[1]) + test_local_compare(self._report_dir_baseline, self._report_dir) + + def test_local_cmp_filter_unres(self): + """ + Filter unresolved results in local compare mode. + """ + def get_filtered_output(filters=None): + diff_cmd = [self._codechecker_cmd, "cmd", "diff", + "--unresolved", + "--url", self._url, + "-b", self._report_dir_baseline, + "-n", self._report_dir + ] + if filters: + diff_cmd.extend(filters) + + print(diff_cmd) + process = subprocess.Popen( + diff_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=self._test_config['codechecker_cfg']['check_env'], + cwd=os.environ['TEST_WORKSPACE']) + out, err = process.communicate() + print(out+err) + return out + + # Filter by severity levels. + res = get_filtered_output(['--severity', 'low']) + self.assertEqual(len(re.findall(r'\[LOW\]', res)), 6) + self.assertEqual(len(re.findall(r'\[HIGH\]', res)), 0) + + res = get_filtered_output(['--severity', 'high']) + self.assertEqual(len(re.findall(r'\[LOW\]', res)), 0) + self.assertEqual(len(re.findall(r'\[HIGH\]', res)), 19) + + res = get_filtered_output(['--severity', 'high', 'low']) + self.assertEqual(len(re.findall(r'\[LOW\]', res)), 6) + self.assertEqual(len(re.findall(r'\[HIGH\]', res)), 19) + + res = get_filtered_output() + self.assertEqual(len(re.findall(r'\[LOW\]', res)), 6) + self.assertEqual(len(re.findall(r'\[HIGH\]', res)), 19) + + # Filter by file path. + res = get_filtered_output(['--file', '*divide_zero.cpp']) + self.assertEqual(len(re.findall(r'divide_zero.cpp', res)), 4) + self.assertEqual(len(re.findall(r'new_delete.cpp', res)), 0) + + res = get_filtered_output(['--file', + 'divide_zero.cpp', # Exact match. + '*new_delete.cpp']) + self.assertEqual(len(re.findall(r'divide_zero.cpp', res)), 0) + self.assertEqual(len(re.findall(r'new_delete.cpp', res)), 6) + + # Filter by checker name. + res = get_filtered_output(['--checker-name', 'core.NullDereference']) + self.assertEqual(len(re.findall(r'core.NullDereference', res)), 4) + + res = get_filtered_output(['--checker-name', 'core.*']) + self.assertEqual(len(re.findall(r'core.*', res)), 14) + + # Filter by checker message (case insensitive). + res = get_filtered_output(['--checker-msg', 'division by*']) + self.assertEqual(len(re.findall(r'Division by.*', res)), 10) + + # Filter by multiple filter set. + res = get_filtered_output(['--file', '*divide_zero.cpp', + '--severity', 'high']) + self.assertEqual(len(re.findall(r'divide_zero.cpp', res)), 2) + self.assertEqual(len(re.findall(r'\[HIGH\]', res)), 2) def test_max_compound_select(self): """