From 30526f837fae5e40e5f0c1239c8794d151eb46d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Umann?= Date: Tue, 11 Jul 2023 13:47:41 +0200 Subject: [PATCH] [cmd] Fix FP annotations in the case of local-remote diffs Source code suppressions seem to work funny (meaning: really buggy) in the context of diffs. An earlier PR, #3944 fixed a similar issue in the local-local diff case: we forgot to take source code suppressions into account for the baseline set. The issue is somewhat worse for local-remote and remote-local diffs, as discussed in issue #3884. As a feature, this should be comically simple: calculate the set of outstanding reports from the "baseline" and the "new" sets. Suppose these sets are called O(baseline) and O(new). Then, calculating diffs should be this easy: * NEW: O(new) - O(baseline) * RESOLVED: O(baseline) - O(new) * UNRESOLVED: The intersection of O(baseline) and O(new) But, for historical and performance reasons, the way we do this is far, far more convoluted. We use two API calls: `getDiffResultsHash` and `getRunResults`. We use the former call to send the list of bug reports (and more specifically, only the bug hashes of those reports) we have locally to the server, and it returns a set of hashes found on the server, ideally those that would be present in the diff set. We constantly change and adjust these sets of hashes with review status rules, source code suppressions, and after some juggling, we ask for the actual report objects (as opposed to simple hashes) via `getRunResults`. Looking at `get_diff_local_dir_remote_run` in `cmd_line_client.py`, the implementation is super complicated, and its hard to judge whether we truly need this much complexity for something that theoretically so simple. Fixing the specific bug of not handling source code suppressions well could be done, on a high level from two approaches: fixing up the API calls on the server side to not return the the hashes that shouldn't be present in the final result, or patch the client to filter them out. This patch presents the client-only approach. Here is my argument for it. We already branch out for older client versions within (#3746), so it would be a chore to fix this for both client versions. You can argue whether fixing this only for later client versions is okay (thats a fair argument), but I think its fair to ask users to upgrade their client if its starts misbehaving. After all, the tool doesn't break, it just work a little funny (meaning: really buggy), which it already does for the local-local case that had to be fixed client side. ANYWAYS. The fix is simple: filter for unreviewed and confirmed reports before calling `getRunResults`. I added new tests for tags just to be sure that it doesn't break anything unrelated, but their behaviour didn't change as a result of this patch. --- .../codechecker_client/cmd_line_client.py | 3 + .../diff_cmdline/test_diff_cmdline.py | 142 ++++++++++++++---- 2 files changed, 113 insertions(+), 32 deletions(-) diff --git a/web/client/codechecker_client/cmd_line_client.py b/web/client/codechecker_client/cmd_line_client.py index a14aeecb89..38ff26a02a 100644 --- a/web/client/codechecker_client/cmd_line_client.py +++ b/web/client/codechecker_client/cmd_line_client.py @@ -268,6 +268,9 @@ def get_diff_base_results( ): """Get the run results from the server.""" report_filter.reportHash = base_hashes + suppressed_hashes + report_filter.reviewStatus = [ + ttypes.ReviewStatus.UNREVIEWED, + ttypes.ReviewStatus.CONFIRMED] sort_mode = [(ttypes.SortMode( ttypes.SortType.FILENAME, diff --git a/web/tests/functional/diff_cmdline/test_diff_cmdline.py b/web/tests/functional/diff_cmdline/test_diff_cmdline.py index 2dd934e666..893079feb5 100644 --- a/web/tests/functional/diff_cmdline/test_diff_cmdline.py +++ b/web/tests/functional/diff_cmdline/test_diff_cmdline.py @@ -380,13 +380,8 @@ def get_run_diff_count(diff_type: DiffType): ["run1"], [dir2], []) return len(reports) - # b() is a new report. self.assertEqual(get_run_diff_count(DiffType.NEW), 0) - - # a() is the old report. self.assertEqual(get_run_diff_count(DiffType.RESOLVED), 0) - - # There are no common reports. self.assertEqual(get_run_diff_count(DiffType.UNRESOLVED), 1) def get_run_diff_count_reverse(diff_type: DiffType): @@ -396,13 +391,8 @@ def get_run_diff_count_reverse(diff_type: DiffType): [dir2], [], ["run1"]) return len(reports) - # b() is a new report. self.assertEqual(get_run_diff_count_reverse(DiffType.NEW), 0) - - # a() is the old report. self.assertEqual(get_run_diff_count_reverse(DiffType.RESOLVED), 0) - - # There are no common reports. self.assertEqual(get_run_diff_count_reverse(DiffType.UNRESOLVED), 1) def test_localFPAnnotated_remote_identical(self): @@ -496,15 +486,8 @@ def get_run_diff_count(diff_type: DiffType): ["run2"], [dir1], []) return len(reports) - # b() is a new report. self.assertEqual(get_run_diff_count(DiffType.NEW), 1) - - # FIXME: This should be 0 -- the "before" set is suppressed, the - # "after" set is no longer, so it should only be in the NEW set. - # Not to mention that these sets must be disjointed. - self.assertEqual(get_run_diff_count(DiffType.RESOLVED), 1) - - # There are no common reports. + self.assertEqual(get_run_diff_count(DiffType.RESOLVED), 0) self.assertEqual(get_run_diff_count(DiffType.UNRESOLVED), 0) def get_run_diff_count_reverse(diff_type: DiffType): @@ -514,15 +497,8 @@ def get_run_diff_count_reverse(diff_type: DiffType): [dir1], [], ["run2"]) return len(reports) - # FIXME: This should be 0 -- the "before" set is suppressed, the - # "after" set is no longer, so it should only be in the RESOLVED set. - # Not to mention that these sets must be disjointed. - self.assertEqual(get_run_diff_count_reverse(DiffType.NEW), 1) - - # a() is the old report. + self.assertEqual(get_run_diff_count_reverse(DiffType.NEW), 0) self.assertEqual(get_run_diff_count_reverse(DiffType.RESOLVED), 1) - - # There are no common reports. self.assertEqual(get_run_diff_count_reverse(DiffType.UNRESOLVED), 0) def test_local_remoteReviewStatusRule_identical(self): @@ -534,7 +510,10 @@ def test_local_remoteReviewStatusRule_identical(self): local run), yet the rule still affects the local run. This implies that review status rules are a timeless property -- once a hash has a rule, all reports matching it before or after the rule - was made are affected. As a result, it should be in the UNRESOLVED set. + was made are affected. + Since neither the report is not present in either of the baseline's, + nor the new run's set outstanding reports, it shouldn't be present in + any of the NEW, RESOLVED or UNRESOLVED sets. """ # Create two identical runs, store one on the server, leave one # locally. @@ -567,8 +546,7 @@ def get_run_diff_count(diff_type: DiffType): return len(reports) self.assertEqual(get_run_diff_count(DiffType.NEW), 0) - # FIXME: This should be in the UNRESOLVED set. - self.assertEqual(get_run_diff_count(DiffType.RESOLVED), 1) + self.assertEqual(get_run_diff_count(DiffType.RESOLVED), 0) self.assertEqual(get_run_diff_count(DiffType.UNRESOLVED), 0) def get_run_diff_count_reverse(diff_type: DiffType): @@ -580,8 +558,7 @@ def get_run_diff_count_reverse(diff_type: DiffType): [dir2], [], ["run1"]) return len(reports) - # FIXME: This should be in the UNRESOLVED set. - self.assertEqual(get_run_diff_count_reverse(DiffType.NEW), 1) + self.assertEqual(get_run_diff_count_reverse(DiffType.NEW), 0) self.assertEqual(get_run_diff_count_reverse(DiffType.RESOLVED), 0) self.assertEqual(get_run_diff_count_reverse(DiffType.UNRESOLVED), 0) @@ -596,8 +573,110 @@ def get_run_diff_count_reverse(diff_type: DiffType): # ===--- Remote-Remote tests in between tags. ------------------------=== # + # Ideally, diffing tags should work the same as diffing two remote runs or + # local directory. + # TODO: remote-remote diffs concerning tags + def test_remoteFPAnnotated_remote_tag(self): + dir1 = os.path.join(self.test_workspace, "dir1") + dir2 = os.path.join(self.test_workspace, "dir2") + + src_div_by_zero_FP = """ +void a() { + int i = 0; + // codechecker_false_positive [all] SUPPRESS ALL + (void)(10 / i); +} +""" + src_div_by_zero = """ +void a() { + int i = 0; + (void)(10 / i); +} +""" + # Note that we're storing under the same run. + self.__analyze_and_store(dir1, "run1", src_div_by_zero_FP, "tag1") + self.__analyze_and_store(dir2, "run1", src_div_by_zero, "tag2") + + report_filter = ReportFilter() + report_filter.reviewStatus = [] + report_filter.detection_status = [] + + def get_run_diff_count(diff_type: DiffType): + reports, _, _ = get_diff_remote_runs( + self._cc_client, report_filter, diff_type, [], + ["run1:tag1"], ["run1:tag2"]) + return len(reports) + + # FIXME: The FP suppression disappeared from one tag to the next, so it + # should be in the NEW set. + self.assertEqual(get_run_diff_count(DiffType.NEW), 0) + self.assertEqual(get_run_diff_count(DiffType.RESOLVED), 0) + self.assertEqual(get_run_diff_count(DiffType.UNRESOLVED), 1) + + def get_run_diff_count_reverse(diff_type: DiffType): + reports, _, _ = get_diff_remote_runs( + self._cc_client, report_filter, diff_type, [], + ["run1:tag1"], ["run1:tag2"]) + return len(reports) + + # FIXME: The FP suppression disappeared from one tag to the next, so it + # should be in the RESOLVED set when the diff sets are reversed. + self.assertEqual(get_run_diff_count_reverse(DiffType.NEW), 0) + self.assertEqual(get_run_diff_count_reverse(DiffType.RESOLVED), 0) + self.assertEqual(get_run_diff_count_reverse(DiffType.UNRESOLVED), 1) + + def test_remote_remoteFPAnnotated_tag(self): + # Diff two different, local runs. + dir1 = os.path.join(self.test_workspace, "dir1") + dir2 = os.path.join(self.test_workspace, "dir2") + + src_div_by_zero = """ +void a() { + int i = 0; + (void)(10 / i); +} +""" + src_div_by_zero_FP = """ +void a() { + int i = 0; + // codechecker_false_positive [all] SUPPRESS ALL + (void)(10 / i); +} +""" + # Note that we're storing under the same run. + self.__analyze_and_store(dir1, "run1", src_div_by_zero, "tag1") + self.__analyze_and_store(dir2, "run1", src_div_by_zero_FP, "tag2") + + report_filter = ReportFilter() + report_filter.reviewStatus = [] + report_filter.detection_status = [] + + def get_run_diff_count(diff_type: DiffType): + reports, _, _ = get_diff_remote_runs( + self._cc_client, report_filter, diff_type, [], + ["run1:tag1"], ["run1:tag2"]) + return len(reports) + + # FIXME: The FP suppression appeared from one tag to the next, so it + # should be in the RESOLVED set. + self.assertEqual(get_run_diff_count(DiffType.NEW), 0) + self.assertEqual(get_run_diff_count(DiffType.RESOLVED), 0) + self.assertEqual(get_run_diff_count(DiffType.UNRESOLVED), 0) + + def get_run_diff_count_reverse(diff_type: DiffType): + reports, _, _ = get_diff_remote_runs( + self._cc_client, report_filter, diff_type, [], + ["run1:tag1"], ["run1:tag2"]) + return len(reports) + + # FIXME: The FP suppression appeared from one tag to the next, so it + # should be in the NEW set when the diff sets are reversed. + self.assertEqual(get_run_diff_count_reverse(DiffType.NEW), 0) + self.assertEqual(get_run_diff_count_reverse(DiffType.RESOLVED), 0) + self.assertEqual(get_run_diff_count_reverse(DiffType.UNRESOLVED), 0) + def test_remote_remote_tag_FixedAtDate(self): """ When a run disappears from one tag to the next, we regard it as fixed, @@ -608,7 +687,6 @@ def test_remote_remote_tag_FixedAtDate(self): You can read more about this bug here: https://github.com/Ericsson/codechecker/pull/3853 """ - # Diff two different, local runs. dir1 = os.path.join(self.test_workspace, "dir1") dir2 = os.path.join(self.test_workspace, "dir2")