Test cases added for local compare mode

libcodechecker/cmd/cmd_line_client.py

@@ -662,7 +662,7 @@ def handle_auth_requests(args):
         saved_auth = session.getAuthString(args.host, args.port)
 
         if saved_auth:
-            LOG.info("Logging in using preconfigured credentials...")
+            LOG.info("Logging in using pre-configured credentials...")
             username = saved_auth.split(":")[0]
             pwd = saved_auth.split(":")[1]
         else:

tests/functional/diff/__init__.py

@@ -68,6 +68,7 @@ def setup_package():
         'skip_list_file': skip_list_file,
         'check_env': test_env,
         'workspace': TEST_WORKSPACE,
+        'reportdir': os.path.join(TEST_WORKSPACE, 'reports'),
         'pg_db_config': pg_db_config,
         'checkers': ['-d', 'core.CallAndMessage',
                      '-e', 'core.StackAddressEscape']
@@ -111,6 +112,15 @@ def setup_package():
         sys.exit(1)
     print("Second analysis of the test project was successful.")
 
+    # Run the second analysis results
+    # into a report directory
+    ret = codechecker.analyze(codechecker_cfg,
+                              test_project_name_new,
+                              test_project_path)
+    if ret:
+        sys.exit(1)
+    print("CodeChecker analyze of test project was successful.")
+
     if pg_db_config:
         print("Waiting for PotgreSQL to stop.")
         codechecker.wait_for_postgres_shutdown(TEST_WORKSPACE)

tests/functional/diff/test_regression_diff.py

@@ -11,7 +11,9 @@
 import unittest
 import logging
 
+import re
 import shared
+import subprocess
 
 from codeCheckerDBAccess.ttypes import DiffType
 from codeCheckerDBAccess.ttypes import ReportFilter
@@ -64,6 +66,11 @@ def setUp(self):
         self._base_runid = test_runs[0].runId  # base
         self._new_runid = test_runs[1].runId  # new
 
+        self._codechecker_cmd = env.codechecker_cmd()
+        self._report_dir = os.path.join(test_workspace, "reports")
+        self._test_config = env.import_test_cfg(test_workspace)
+        self._run_names = env.get_run_names(test_workspace)
+
     def test_get_diff_res_count_new(self):
         """
         Count the new results with no filter.
@@ -231,3 +238,88 @@ def test_get_diff_res_types_unresolved_filter(self):
                 self.assertEqual(len(diff_res), 1)
                 self.assertEqual(test_result_count, diff_res[0].count)
                 self.assertEqual(checker_name, diff_res[0].checkerId)
+
+    def test_local_compare_res_count_new(self):
+        """
+        Count the new results with no filter in local compare mode.
+        """
+        base_run_name = self._run_names[0]
+        vh = self._test_config['codechecker_cfg']['viewer_host']
+        vp = self._test_config['codechecker_cfg']['viewer_port']
+        diff_cmd = [self._codechecker_cmd, "cmd", "diff",
+                    "--new",
+                    "--host", vh,
+                    "--port", str(vp),
+                    "-b", base_run_name,
+                    "-n", self._report_dir
+                    ]
+        print(diff_cmd)
+        process = subprocess.Popen(
+                    diff_cmd, stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE, cwd=os.environ['TEST_WORKSPACE'])
+        out, err = process.communicate()
+        print(out+err)
+
+        # 5 new core.CallAndMessage issues.
+        count = len(re.findall(r'\[core\.CallAndMessage\]', out))
+        self.assertEqual(count, 5)
+
+    def test_local_compare_res_count_resovled(self):
+        """
+        Count the resolved results with no filter in local compare mode.
+        """
+        base_run_name = self._run_names[0]
+        vh = self._test_config['codechecker_cfg']['viewer_host']
+        vp = self._test_config['codechecker_cfg']['viewer_port']
+        diff_cmd = [self._codechecker_cmd, "cmd", "diff",
+                    "--resolved",
+                    "--host", vh,
+                    "--port", str(vp),
+                    "-b", base_run_name,
+                    "-n", self._report_dir
+                    ]
+        print(diff_cmd)
+        process = subprocess.Popen(
+                    diff_cmd, stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE, cwd=os.environ['TEST_WORKSPACE'])
+        out, err = process.communicate()
+        print(out+err)
+
+        # # 3 disappeared core.StackAddressEscape issues
+        count = len(re.findall(r'\[core\.StackAddressEscape\]', out))
+        self.assertEqual(count, 3)
+
+    def test_local_compare_res_count_unresovled(self):
+        """
+        Count the unresolved results with no filter in local compare mode.
+        """
+        base_run_name = self._run_names[0]
+        vh = self._test_config['codechecker_cfg']['viewer_host']
+        vp = self._test_config['codechecker_cfg']['viewer_port']
+        diff_cmd = [self._codechecker_cmd, "cmd", "diff",
+                    "--unresolved",
+                    "--host", vh,
+                    "--port", str(vp),
+                    "-b", base_run_name,
+                    "-n", self._report_dir
+                    ]
+        print(diff_cmd)
+        process = subprocess.Popen(
+                    diff_cmd, stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE, cwd=os.environ['TEST_WORKSPACE'])
+        out, err = process.communicate()
+        print(out+err)
+
+        # # 3 disappeared core.StackAddressEscape issues
+        count = len(re.findall(r'\[core\.DivideZero\]', out))
+        self.assertEqual(count, 3)
+        count = len(re.findall(r'\[deadcode\.DeadStores\]', out))
+        self.assertEqual(count, 5)
+        count = len(re.findall(r'\[core\.NullDereference\]', out))
+        self.assertEqual(count, 3)
+        count = len(re.findall(r'\[cplusplus\.NewDelete\]', out))
+        self.assertEqual(count, 5)
+        count = len(re.findall(r'\[unix\.Malloc\]', out))
+        self.assertEqual(count, 1)
+        count = len(re.findall(r'\[core.DivideZero\]', out))
+        self.assertEqual(count, 3)

tests/libtest/codechecker.py

@@ -81,6 +81,56 @@ def check(codechecker_cfg, test_project_name, test_project_path):
         return cerr.returncode
 
 
+def analyze(codechecker_cfg, test_project_name, test_project_path):
+    """
+    Analyze a test project.
+
+    :checkers parameter should be a list of enabled or disabled checkers
+    Example: ['-d', 'deadcode.DeadStores']
+
+    """
+
+    build_cmd = project.get_build_cmd(test_project_path)
+    build_json = os.path.join(codechecker_cfg['workspace'], "build.json")
+
+    log_cmd = ['CodeChecker', 'log',
+               '-o', build_json,
+               '-b', "'" + build_cmd + "'",
+               ]
+
+    analyze_cmd = ['CodeChecker', 'analyze',
+                   build_json,
+                   '-o', codechecker_cfg['reportdir'],
+                   '--analyzers', 'clangsa'
+                   ]
+
+    analyze_cmd.extend(codechecker_cfg['checkers'])
+    try:
+        print("LOG:")
+        proc = subprocess.Popen(shlex.split(' '.join(log_cmd)),
+                                cwd=test_project_path,
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE,
+                                env=codechecker_cfg['check_env'])
+        out, err = proc.communicate()
+        print(out)
+        print(err)
+
+        print("ANALYZE:")
+        print(shlex.split(' '.join(analyze_cmd)))
+        proc = subprocess.Popen(shlex.split(' '.join(analyze_cmd)),
+                                cwd=test_project_path,
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE,
+                                env=codechecker_cfg['check_env'])
+        out, err = proc.communicate()
+        print(out)
+        print(err)
+        return 0
+    except CalledProcessError as cerr:
+        print("Failed to call:\n" + ' '.join(cerr.cmd))
+        return cerr.returncode
+
 def serv_cmd(codechecker_cfg, test_config):
 
     server_cmd = ['CodeChecker', 'server',