From 44c61cf27fb3a3d689dcc3cf9a95e73d3882e111 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Umann?= Date: Tue, 24 Oct 2023 15:04:37 +0200 Subject: [PATCH 1/4] [analyzer] Add support the CC_ANALYZER_BIN env var This environmental variable can be used the specify the absolute path of an analyzer. Similar to #4041, but this is the only solution we can realistically get though the door. --- .../analyzers/analyzer_types.py | 5 ++- analyzer/codechecker_analyzer/arg.py | 17 ++++++++ analyzer/codechecker_analyzer/cmd/analyze.py | 42 ++++++++++++++++++- docs/analyzer/user_guide.md | 10 +++++ 4 files changed, 72 insertions(+), 2 deletions(-) diff --git a/analyzer/codechecker_analyzer/analyzers/analyzer_types.py b/analyzer/codechecker_analyzer/analyzers/analyzer_types.py index 8882e38c9f..060a89a7f8 100644 --- a/analyzer/codechecker_analyzer/analyzers/analyzer_types.py +++ b/analyzer/codechecker_analyzer/analyzers/analyzer_types.py @@ -189,7 +189,10 @@ def check_supported_analyzers(analyzers): error = analyzer.is_binary_version_incompatible(check_env) if error: failed_analyzers.add((analyzer_name, - f"Incompatible version: {error}")) + f"Incompatible version: {error} " + "Maybe try setting an absolute path to " + "a different analyzer binary via the " + "env variable CC_ANALYZER_BIN?")) available_analyzer = False if not analyzer_bin or \ diff --git a/analyzer/codechecker_analyzer/arg.py b/analyzer/codechecker_analyzer/arg.py index afdc664dcc..6d49cecabf 100644 --- a/analyzer/codechecker_analyzer/arg.py +++ b/analyzer/codechecker_analyzer/arg.py @@ -18,6 +18,8 @@ 'AnalyzerConfig', ["analyzer", "option", "value"]) CheckerConfig = collections.namedtuple( "CheckerConfig", ["analyzer", "checker", "option", "value"]) +AnalyzerBinary = collections.namedtuple( + "AnalyzerBinary", ["analyzer", "path"]) class OrderedCheckersAction(argparse.Action): @@ -133,3 +135,18 @@ def checker_config(arg: str) -> CheckerConfig: return CheckerConfig( m.group("analyzer"), m.group("checker"), m.group("option"), m.group("value")) + + +def analyzer_binary(arg: str) -> AnalyzerBinary: + """ + This function can be used at "type" argument of argparse.add_argument(). + It checks the format of --analyzer_binary flag: : + """ + m = re.match(r"(?P.+):(?P.+)", arg) + + if not m: + raise argparse.ArgumentTypeError( + f"Analyzer binary in wrong format: {arg}, should be " + ":") + + return AnalyzerBinary(m.group("analyzer"), m.group("path")) diff --git a/analyzer/codechecker_analyzer/cmd/analyze.py b/analyzer/codechecker_analyzer/cmd/analyze.py index dd3fe1f8fe..3ea27ecb63 100644 --- a/analyzer/codechecker_analyzer/cmd/analyze.py +++ b/analyzer/codechecker_analyzer/cmd/analyze.py @@ -27,7 +27,7 @@ from codechecker_analyzer.analyzers import analyzer_types, clangsa from codechecker_analyzer.arg import \ OrderedCheckersAction, OrderedConfigAction, existing_abspath, \ - analyzer_config, checker_config + analyzer_config, checker_config, analyzer_binary from codechecker_analyzer.buildlog import log_parser from codechecker_common import arg, logger, cmd_config @@ -44,6 +44,11 @@ epilog_env_var = f""" CC_ANALYZERS_FROM_PATH Set to `yes` or `1` to enforce taking the analyzers from the `PATH` instead of the given binaries. + CC_ANALYZER_BIN Set the absolute paths of an analyzer binaries. + Overrides other means of CodeChecker getting hold of + binary. + Format: CC_ANALYZER_BIN=':/path/to/bin1; + :/path/to/bin2' CC_CLANGSA_PLUGIN_DIR If the CC_ANALYZERS_FROM_PATH environment variable is set you can configure the plugin directory of the Clang Static Analyzer by using this environment @@ -899,6 +904,40 @@ def __get_result_source_files(metadata): return result_src_files +def __parse_CC_ANALYZER_BIN(): + context = analyzer_context.get_context() + if 'CC_ANALYZER_BIN' in context.analyzer_env: + had_error = False + for value in context.analyzer_env['CC_ANALYZER_BIN'].split(';'): + try: + analyzer_name, path = analyzer_binary(value) + except argparse.ArgumentTypeError as e: + LOG.error(e) + had_error = True + continue + + if analyzer_name not in analyzer_types.supported_analyzers: + LOG.error(f"Unsupported analyzer_name name '{analyzer_name}' " + "given to CC_ANALYZER_BIN!") + had_error = True + if not os.path.isfile(path): + LOG.error(f"'{path}' is not a path to an analyzer binary " + "given to CC_ANALYZER_BIN!") + had_error = True + + if had_error: + continue + + LOG.info(f"Using '{path}' for analyzer '{analyzer_name}'") + context.analyzer_binaries[analyzer_name] = path + + if had_error: + LOG.info("The value of CC_ANALYZER_BIN should be in the format of " + "CC_ANALYZER_BIN=':/path/to/bin1;" + ":/path/to/bin2'") + sys.exit(1) + + def main(args): """ Perform analysis on the given inputs. Possible inputs are a compilation @@ -981,6 +1020,7 @@ def main(args): ctu_or_stats_enabled = True context = analyzer_context.get_context() + __parse_CC_ANALYZER_BIN() # Number of all the compilation commands in the parsed log files, # logged by the logger. diff --git a/docs/analyzer/user_guide.md b/docs/analyzer/user_guide.md index ca48db0c28..8c59b088f0 100644 --- a/docs/analyzer/user_guide.md +++ b/docs/analyzer/user_guide.md @@ -499,6 +499,11 @@ Environment variables for 'CodeChecker analyze' command: CC_ANALYZERS_FROM_PATH Set to `yes` or `1` to enforce taking the analyzers from the `PATH` instead of the given binaries. + CC_ANALYZER_BIN Set the absolute paths of an analyzer binaries. + Overrides other means of CodeChecker getting hold of + binary. + Format: CC_ANALYZER_BIN=':/path/to/bin1; + :/path/to/bin2' CC_CLANGSA_PLUGIN_DIR If the CC_ANALYZERS_FROM_PATH environment variable is set you can configure the plugin directory of the Clang Static Analyzer by using this environment @@ -1031,6 +1036,11 @@ Environment variables CC_ANALYZERS_FROM_PATH Set to `yes` or `1` to enforce taking the analyzers from the `PATH` instead of the given binaries. + CC_ANALYZER_BIN Set the absolute paths of an analyzer binaries. + Overrides other means of CodeChecker getting hold of + binary. + Format: CC_ANALYZER_BIN=':/path/to/bin1; + :/path/to/bin2' CC_CLANGSA_PLUGIN_DIR If the CC_ANALYZERS_FROM_PATH environment variable is set you can configure the plugin directory of the Clang Static Analyzer by using this environment From a31ff167623b47ca178ebef77a47d83a83060149 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Umann?= Date: Tue, 24 Oct 2023 16:14:00 +0200 Subject: [PATCH 2/4] Move the implementation to Context --- .../codechecker_analyzer/analyzer_context.py | 38 +++++++++++++++++++ analyzer/codechecker_analyzer/cmd/analyze.py | 35 ----------------- 2 files changed, 38 insertions(+), 35 deletions(-) diff --git a/analyzer/codechecker_analyzer/analyzer_context.py b/analyzer/codechecker_analyzer/analyzer_context.py index c55c66dabf..1e0ad74a68 100644 --- a/analyzer/codechecker_analyzer/analyzer_context.py +++ b/analyzer/codechecker_analyzer/analyzer_context.py @@ -20,6 +20,7 @@ from pathlib import Path from codechecker_common import logger +from codechecker_analyzer.arg import analyzer_binary from codechecker_common.checker_labels import CheckerLabels from codechecker_common.singleton import Singleton from codechecker_common.util import load_json @@ -90,6 +91,38 @@ def __init__(self): self.__populate_analyzers() self.__populate_replacer() + + def __parse_CC_ANALYZER_BIN(self): + env_var_bins = {} + if 'CC_ANALYZER_BIN' in self.__analyzer_env: + had_error = False + for value in self.__analyzer_env['CC_ANALYZER_BIN'].split(';'): + try: + analyzer_name, path = analyzer_binary(value) + except argparse.ArgumentTypeError as e: + LOG.error(e) + had_error = True + continue + + if not os.path.isfile(path): + LOG.error(f"'{path}' is not a path to an analyzer binary " + "given to CC_ANALYZER_BIN!") + had_error = True + + if had_error: + continue + + LOG.info(f"Using '{path}' for analyzer '{analyzer_name}'") + env_var_bins[analyzer_name] = path + + if had_error: + LOG.info("The value of CC_ANALYZER_BIN should be in the" + "format of " + "CC_ANALYZER_BIN=':/path/to/bin1;" + ":/path/to/bin2'") + sys.exit(1) + return env_var_bins + def __get_package_config(self): """ Get package configuration. """ pckg_config_file = os.path.join( @@ -166,8 +199,13 @@ def __populate_analyzers(self): if not analyzer_from_path: analyzer_env = self.analyzer_env + env_var_bin = self.__parse_CC_ANALYZER_BIN() + compiler_binaries = self.pckg_layout.get('analyzers') for name, value in compiler_binaries.items(): + if name in env_var_bin: + self.__analyzers[name] = env_var_bin[name] + continue if analyzer_from_path: value = os.path.basename(value) diff --git a/analyzer/codechecker_analyzer/cmd/analyze.py b/analyzer/codechecker_analyzer/cmd/analyze.py index 3ea27ecb63..9251c3a221 100644 --- a/analyzer/codechecker_analyzer/cmd/analyze.py +++ b/analyzer/codechecker_analyzer/cmd/analyze.py @@ -904,40 +904,6 @@ def __get_result_source_files(metadata): return result_src_files -def __parse_CC_ANALYZER_BIN(): - context = analyzer_context.get_context() - if 'CC_ANALYZER_BIN' in context.analyzer_env: - had_error = False - for value in context.analyzer_env['CC_ANALYZER_BIN'].split(';'): - try: - analyzer_name, path = analyzer_binary(value) - except argparse.ArgumentTypeError as e: - LOG.error(e) - had_error = True - continue - - if analyzer_name not in analyzer_types.supported_analyzers: - LOG.error(f"Unsupported analyzer_name name '{analyzer_name}' " - "given to CC_ANALYZER_BIN!") - had_error = True - if not os.path.isfile(path): - LOG.error(f"'{path}' is not a path to an analyzer binary " - "given to CC_ANALYZER_BIN!") - had_error = True - - if had_error: - continue - - LOG.info(f"Using '{path}' for analyzer '{analyzer_name}'") - context.analyzer_binaries[analyzer_name] = path - - if had_error: - LOG.info("The value of CC_ANALYZER_BIN should be in the format of " - "CC_ANALYZER_BIN=':/path/to/bin1;" - ":/path/to/bin2'") - sys.exit(1) - - def main(args): """ Perform analysis on the given inputs. Possible inputs are a compilation @@ -1020,7 +986,6 @@ def main(args): ctu_or_stats_enabled = True context = analyzer_context.get_context() - __parse_CC_ANALYZER_BIN() # Number of all the compilation commands in the parsed log files, # logged by the logger. From 34a1601fa386af06dc1c2a97266c27f258e2fbf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Umann?= Date: Thu, 26 Oct 2023 09:39:38 +0200 Subject: [PATCH 3/4] Make pylint and pycodestyle happy --- analyzer/codechecker_analyzer/analyzer_context.py | 4 ++-- analyzer/codechecker_analyzer/cmd/analyze.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/analyzer/codechecker_analyzer/analyzer_context.py b/analyzer/codechecker_analyzer/analyzer_context.py index 1e0ad74a68..41fd167619 100644 --- a/analyzer/codechecker_analyzer/analyzer_context.py +++ b/analyzer/codechecker_analyzer/analyzer_context.py @@ -12,6 +12,7 @@ # pylint: disable=no-name-in-module from distutils.spawn import find_executable +from argparse import ArgumentTypeError import os import platform @@ -91,7 +92,6 @@ def __init__(self): self.__populate_analyzers() self.__populate_replacer() - def __parse_CC_ANALYZER_BIN(self): env_var_bins = {} if 'CC_ANALYZER_BIN' in self.__analyzer_env: @@ -99,7 +99,7 @@ def __parse_CC_ANALYZER_BIN(self): for value in self.__analyzer_env['CC_ANALYZER_BIN'].split(';'): try: analyzer_name, path = analyzer_binary(value) - except argparse.ArgumentTypeError as e: + except ArgumentTypeError as e: LOG.error(e) had_error = True continue diff --git a/analyzer/codechecker_analyzer/cmd/analyze.py b/analyzer/codechecker_analyzer/cmd/analyze.py index 9251c3a221..320a8a8426 100644 --- a/analyzer/codechecker_analyzer/cmd/analyze.py +++ b/analyzer/codechecker_analyzer/cmd/analyze.py @@ -27,7 +27,7 @@ from codechecker_analyzer.analyzers import analyzer_types, clangsa from codechecker_analyzer.arg import \ OrderedCheckersAction, OrderedConfigAction, existing_abspath, \ - analyzer_config, checker_config, analyzer_binary + analyzer_config, checker_config from codechecker_analyzer.buildlog import log_parser from codechecker_common import arg, logger, cmd_config From 1ef096cb7ba130d9d7b151ae96bcf31615cf8b42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=B3f=20Umann?= Date: Thu, 26 Oct 2023 15:19:24 +0200 Subject: [PATCH 4/4] Fix a crash where analyzer binaries were absent AnalyzerContext keeps track of the binaries for each analyzer. However, if the binary isn't found, we don't map None to the respective analyzer, we simply skip it. This lead to key errors when we called the version getter function, which I fixed in this patch. --- analyzer/codechecker_analyzer/analyzer_context.py | 1 + analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py | 4 ++++ analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py | 4 ++++ analyzer/codechecker_analyzer/analyzers/cppcheck/analyzer.py | 3 +++ analyzer/codechecker_analyzer/analyzers/gcc/analyzer.py | 3 +++ 5 files changed, 15 insertions(+) diff --git a/analyzer/codechecker_analyzer/analyzer_context.py b/analyzer/codechecker_analyzer/analyzer_context.py index 41fd167619..9ff3363011 100644 --- a/analyzer/codechecker_analyzer/analyzer_context.py +++ b/analyzer/codechecker_analyzer/analyzer_context.py @@ -220,6 +220,7 @@ def __populate_analyzers(self): if not compiler_binary: LOG.debug("'%s' binary can not be found in your PATH!", value) + self.__analyzers[name] = None continue self.__analyzers[name] = os.path.realpath(compiler_binary) diff --git a/analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py b/analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py index 02b32580fa..befcfb7e72 100644 --- a/analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py +++ b/analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py @@ -172,6 +172,10 @@ def __add_plugin_load_flags(cls, analyzer_cmd: List[str]): @classmethod def get_binary_version(self, environ, details=False) -> str: + # No need to LOG here, we will emit a warning later anyway. + if not self.analyzer_binary(): + return None + if details: version = [self.analyzer_binary(), '--version'] else: diff --git a/analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py b/analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py index 5d012750b5..4610823508 100644 --- a/analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py +++ b/analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py @@ -231,6 +231,10 @@ def analyzer_binary(cls): @classmethod def get_binary_version(self, environ, details=False) -> str: + # No need to LOG here, we will emit a warning later anyway. + if not self.analyzer_binary(): + return None + version = [self.analyzer_binary(), '--version'] try: output = subprocess.check_output(version, diff --git a/analyzer/codechecker_analyzer/analyzers/cppcheck/analyzer.py b/analyzer/codechecker_analyzer/analyzers/cppcheck/analyzer.py index c8cfbc535f..d4b9e6ad96 100644 --- a/analyzer/codechecker_analyzer/analyzers/cppcheck/analyzer.py +++ b/analyzer/codechecker_analyzer/analyzers/cppcheck/analyzer.py @@ -85,6 +85,9 @@ def analyzer_binary(cls): @classmethod def get_binary_version(self, environ, details=False) -> str: """ Get analyzer version information. """ + # No need to LOG here, we will emit a warning later anyway. + if not self.analyzer_binary(): + return None version = [self.analyzer_binary(), '--version'] try: output = subprocess.check_output(version, diff --git a/analyzer/codechecker_analyzer/analyzers/gcc/analyzer.py b/analyzer/codechecker_analyzer/analyzers/gcc/analyzer.py index 39c3c0ead3..00358e9724 100644 --- a/analyzer/codechecker_analyzer/analyzers/gcc/analyzer.py +++ b/analyzer/codechecker_analyzer/analyzers/gcc/analyzer.py @@ -173,6 +173,9 @@ def get_binary_version(self, environ, details=False) -> str: """ Return the analyzer version. """ + # No need to LOG here, we will emit a warning later anyway. + if not self.analyzer_binary(): + return None if details: version = [self.analyzer_binary(), '--version'] else: