From 1d1b8820ef6f197ff90ba7562845b4a6f2f5801b Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 18 Sep 2024 14:53:21 +0200 Subject: [PATCH 01/33] add SearchPaths variable class for colon-separated lists of absolute paths --- easybuild/tools/toolchain/variables.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/easybuild/tools/toolchain/variables.py b/easybuild/tools/toolchain/variables.py index 482f989480..5e051d4331 100644 --- a/easybuild/tools/toolchain/variables.py +++ b/easybuild/tools/toolchain/variables.py @@ -46,6 +46,11 @@ class LinkLibraryPaths(AbsPathList): PREFIX = '-L' +class SearchPaths(AbsPathList): + """Colon-separated list of absolute paths""" + SEPARATOR = ':' + + class FlagList(StrList): """Flag list""" PREFIX = "-" From 1d15f4d14369fe0594c3d3f2628265f442f23b78 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 18 Sep 2024 14:55:22 +0200 Subject: [PATCH 02/33] add supported search paths by GNU CPP as toolchain variables --- easybuild/tools/toolchain/constants.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/toolchain/constants.py b/easybuild/tools/toolchain/constants.py index 643e8b4d21..72b333cf43 100644 --- a/easybuild/tools/toolchain/constants.py +++ b/easybuild/tools/toolchain/constants.py @@ -31,9 +31,9 @@ * Kenneth Hoste (Ghent University) """ +from easybuild.tools.toolchain.variables import CommandFlagList, CommaSharedLibs, CommaStaticLibs, FlagList +from easybuild.tools.toolchain.variables import IncludePaths, LibraryList, LinkLibraryPaths, SearchPaths from easybuild.tools.variables import AbsPathList -from easybuild.tools.toolchain.variables import CommandFlagList, CommaSharedLibs, CommaStaticLibs -from easybuild.tools.toolchain.variables import FlagList, IncludePaths, LibraryList, LinkLibraryPaths COMPILER_VARIABLES = [ @@ -65,7 +65,13 @@ ('LDFLAGS', 'Flags passed to linker'), # TODO: overridden by command line? ], IncludePaths: [ - ('CPPFLAGS', 'Precompiler flags'), + ('CPPFLAGS', 'Preprocessor flags'), + ], + SearchPaths: [ + ('CPATH', 'Location of C/C++ header files'), + ('C_INCLUDE_PATH', 'Location of C header files'), + ('CPLUS_INCLUDE_PATH', 'Location of C++ header files'), + ('OBJC_INCLUDE_PATH', 'Location of Objective C header files'), ], CommandFlagList: COMPILER_VARIABLES, } From 841584f1629cfa0bbb16e52b959338e1df88cb2c Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 18 Sep 2024 14:57:09 +0200 Subject: [PATCH 03/33] add build option cpp-headers-search-paths --- easybuild/tools/config.py | 1 + easybuild/tools/options.py | 3 +++ easybuild/tools/toolchain/toolchain.py | 11 +++++++++++ 3 files changed, 15 insertions(+) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 80f5b5bc75..8f07f298b4 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -206,6 +206,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'backup_modules', 'banned_linked_shared_libs', 'checksum_priority', + 'cpp_headers_search_path', 'container_config', 'container_image_format', 'container_image_name', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 8921ce482c..685e84a26e 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -101,6 +101,7 @@ from easybuild.tools.run import run_shell_cmd from easybuild.tools.package.utilities import avail_package_naming_schemes from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, OPTARCH_MAP_CHAR, OPTARCH_SEP, Compiler +from easybuild.tools.toolchain.toolchain import CPP_HEADER_SEARCH_PATHS, DEFAULT_CPP_HEADER_SEARCH_PATH from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME from easybuild.tools.repository.repository import avail_repositories from easybuild.tools.systemtools import DARWIN, UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family @@ -375,6 +376,8 @@ def override_options(self): 'choice', 'store_or_None', DEFAULT_CHECKSUM_PRIORITY, CHECKSUM_PRIORITY_CHOICES), 'cleanup-builddir': ("Cleanup build dir after successful installation.", None, 'store_true', True), 'cleanup-tmpdir': ("Cleanup tmp dir after successful run.", None, 'store_true', True), + 'cpp-headers-search-path': ("Search path used by EasyBuild to inject include directories", 'choice', + 'store', DEFAULT_CPP_HEADER_SEARCH_PATH, [*CPP_HEADER_SEARCH_PATHS]), 'color': ("Colorize output", 'choice', 'store', fancylogger.Colorize.AUTO, fancylogger.Colorize, {'metavar': 'WHEN'}), 'consider-archived-easyconfigs': ("Also consider archived easyconfigs", None, 'store_true', False), diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 3dd76903b6..28e1c1b5c0 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -95,6 +95,17 @@ TOOLCHAIN_CAPABILITY_LAPACK_FAMILY, TOOLCHAIN_CAPABILITY_MPI_FAMILY, ] +# modes to handle CPP header search paths +# see: https://gcc.gnu.org/onlinedocs/cpp/Environment-Variables.html +CPP_HEADER_SEARCH_PATH_FLAGS = "CPPFLAGS" +CPP_HEADER_SEARCH_PATH_CPATH = "CPATH" +CPP_HEADER_SEARCH_PATH_INCLUDE = "INCLUDE_PATHS" +CPP_HEADER_SEARCH_PATHS = { + CPP_HEADER_SEARCH_PATH_FLAGS: ["CPPFLAGS"], + CPP_HEADER_SEARCH_PATH_CPATH: ["CPATH"], + CPP_HEADER_SEARCH_PATH_INCLUDE: ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"], +} +DEFAULT_CPP_HEADER_SEARCH_PATH = CPP_HEADER_SEARCH_PATH_FLAGS def is_system_toolchain(tc_name): From e3163ac1ddc7f14d90af225d9c17abc309e8a35b Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 18 Sep 2024 14:57:21 +0200 Subject: [PATCH 04/33] add toolchain option cpp-headers-search-paths --- easybuild/tools/toolchain/compiler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index de69214388..cf67003570 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -93,6 +93,7 @@ class Compiler(Toolchain): 'vectorize': (None, "Enable compiler auto-vectorization, default except for noopt and lowopt"), 'packed-linker-options': (False, "Pack the linker options as comma separated list"), # ScaLAPACK mainly 'rpath': (True, "Use RPATH wrappers when --rpath is enabled in EasyBuild configuration"), + 'cpp-headers-search-path': (None, "Search path used by EasyBuild to inject include directories"), 'extra_cflags': (None, "Specify extra CFLAGS options."), 'extra_cxxflags': (None, "Specify extra CXXFLAGS options."), 'extra_fflags': (None, "Specify extra FFLAGS options."), From a82bee5bd0173f49e22ef04331c79f4c14a7a21f Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 18 Sep 2024 14:58:49 +0200 Subject: [PATCH 05/33] add paths to headers from dependencies according to the option set in cpp-headers-search-path --- easybuild/tools/toolchain/toolchain.py | 83 +++++++++++++++++--------- 1 file changed, 55 insertions(+), 28 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 28e1c1b5c0..39e8c0e663 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -861,7 +861,7 @@ def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True, else: self.log.debug("prepare: set additional variables onlymod=%s", onlymod) - # add LDFLAGS and CPPFLAGS from dependencies to self.vars + # add linker and preprocessor paths to dependencies to self.vars self._add_dependency_variables() self.generate_vars() self._setenv_variables(onlymod, verbose=not silent) @@ -1060,39 +1060,66 @@ def handle_sysroot(self): setvar('PKG_CONFIG_PATH', os.pathsep.join(pkg_config_path)) def _add_dependency_variables(self, names=None, cpp=None, ld=None): - """ Add LDFLAGS and CPPFLAGS to the self.variables based on the dependencies - names should be a list of strings containing the name of the dependency """ - cpp_paths = ['include'] - ld_paths = ['lib64', 'lib'] - - if cpp is not None: - for p in cpp: - if p not in cpp_paths: - cpp_paths.append(p) - if ld is not None: - for p in ld: - if p not in ld_paths: - ld_paths.append(p) - - if not names: - deps = self.dependencies - else: - deps = [{'name': name} for name in names if name is not None] + Add linker and preprocessor paths to dependencies to self.variables + :names: list of strings containing the name of the dependency + """ + # collect dependencies + dependencies = self.dependencies if names is None else [{"name": name} for name in names if name] # collect software install prefixes for dependencies - roots = [] - for dep in deps: - if dep.get('external_module', False): + dependency_roots = [] + for dep in dependencies: + if dep.get("external_module", False): # for software names provided via external modules, install prefix may be unknown - names = dep['external_module_metadata'].get('name', []) - roots.extend([root for root in self.get_software_root(names) if root is not None]) + names = dep["external_module_metadata"].get("name", []) + dependency_roots.extend([root for root in self.get_software_root(names) if root is not None]) else: - roots.extend(self.get_software_root(dep['name'])) + dependency_roots.extend(self.get_software_root(dep["name"])) + + for root in dependency_roots: + self._add_dependency_cpp_headers(root, extra_dirs=cpp) + self._add_dependency_linker_paths(root, extra_dirs=ld) + + def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None): + """ + Append prepocessor paths for given dependency root directory + """ + header_dirs = ["include"] + if isinstance(extra_dirs, (tuple, list)): + header_dirs.extend(extra_dirs) + header_dirs = set(header_dirs) # remove duplicates + + # mode of operation is defined by cpp-headers-search-path option + # toolchain option has precedence over build option + cpp_headers_opt = self.options.option("cpp-headers-search-path") + if cpp_headers_opt in CPP_HEADER_SEARCH_PATHS: + self.log.debug("cpp-headers-search-path set by toolchain option: %s", cpp_headers_opt) + elif cpp_headers_opt is not None: + raise EasyBuildError( + "Unknown value selected for toolchain option cpp-headers-search-path. Choose one of: %s", + ", ".join(CPP_HEADER_SEARCH_PATHS) + ) + else: + cpp_headers_opt = build_option("cpp_headers_search_path") + self.log.debug("cpp-headers-search-path set by build option: %s", cpp_headers_opt) + + for env_var in CPP_HEADER_SEARCH_PATHS[cpp_headers_opt]: + self.log.debug("Adding header paths to toolchain variable '%s': %s", env_var, dep_root) + self.variables.append_subdirs(env_var, dep_root, subdirs=header_dirs) - for root in roots: - self.variables.append_subdirs("CPPFLAGS", root, subdirs=cpp_paths) - self.variables.append_subdirs("LDFLAGS", root, subdirs=ld_paths) + def _add_dependency_linker_paths(self, dep_root, extra_dirs=None): + """ + Append linker paths for given dependency root directory + """ + lib_dirs = ["lib64", "lib"] + if isinstance(extra_dirs, (tuple, list)): + lib_dirs.extend(extra_dirs) + lib_dirs = set(lib_dirs) # remove duplicates + + env_var = "LDFLAGS" + self.log.debug("Adding lib paths to toolchain variable '%s': %s", env_var, dep_root) + self.variables.append_subdirs(env_var, dep_root, subdirs=lib_dirs) def _setenv_variables(self, donotset=None, verbose=True): """Actually set the environment variables""" From 20647b7c3c2ea057b499822f0cfaffb6d389214f Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 18 Sep 2024 16:04:36 +0200 Subject: [PATCH 06/33] fix case where cpp-headers-search-path is set to None --- easybuild/tools/toolchain/toolchain.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 39e8c0e663..73b78dd73a 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1092,19 +1092,23 @@ def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None): # mode of operation is defined by cpp-headers-search-path option # toolchain option has precedence over build option - cpp_headers_opt = self.options.option("cpp-headers-search-path") - if cpp_headers_opt in CPP_HEADER_SEARCH_PATHS: - self.log.debug("cpp-headers-search-path set by toolchain option: %s", cpp_headers_opt) - elif cpp_headers_opt is not None: + cpp_headers_mode = DEFAULT_CPP_HEADER_SEARCH_PATH + tc_opt = self.options.option("cpp-headers-search-path") + if tc_opt in CPP_HEADER_SEARCH_PATHS: + self.log.debug("cpp-headers-search-path set by toolchain option: %s", cpp_headers_mode) + cpp_headers_mode = tc_opt + elif tc_opt is not None: raise EasyBuildError( "Unknown value selected for toolchain option cpp-headers-search-path. Choose one of: %s", ", ".join(CPP_HEADER_SEARCH_PATHS) ) else: - cpp_headers_opt = build_option("cpp_headers_search_path") - self.log.debug("cpp-headers-search-path set by build option: %s", cpp_headers_opt) + build_opt = build_option("cpp_headers_search_path") + if build_opt is not None: + cpp_headers_mode = build_opt + self.log.debug("cpp-headers-search-path set by build option: %s", cpp_headers_mode) - for env_var in CPP_HEADER_SEARCH_PATHS[cpp_headers_opt]: + for env_var in CPP_HEADER_SEARCH_PATHS[cpp_headers_mode]: self.log.debug("Adding header paths to toolchain variable '%s': %s", env_var, dep_root) self.variables.append_subdirs(env_var, dep_root, subdirs=header_dirs) From d8ab393981f4ea0af15944cbd96b8de9f90e6b32 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 18 Sep 2024 16:05:39 +0200 Subject: [PATCH 07/33] rename CPP_HEADER_* globals to CPP_HEADERS_* --- easybuild/tools/options.py | 4 ++-- easybuild/tools/toolchain/toolchain.py | 24 ++++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 685e84a26e..515bc535b1 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -101,7 +101,7 @@ from easybuild.tools.run import run_shell_cmd from easybuild.tools.package.utilities import avail_package_naming_schemes from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, OPTARCH_MAP_CHAR, OPTARCH_SEP, Compiler -from easybuild.tools.toolchain.toolchain import CPP_HEADER_SEARCH_PATHS, DEFAULT_CPP_HEADER_SEARCH_PATH +from easybuild.tools.toolchain.toolchain import CPP_HEADERS_SEARCH_PATHS, DEFAULT_CPP_HEADERS_SEARCH_PATH from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME from easybuild.tools.repository.repository import avail_repositories from easybuild.tools.systemtools import DARWIN, UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family @@ -377,7 +377,7 @@ def override_options(self): 'cleanup-builddir': ("Cleanup build dir after successful installation.", None, 'store_true', True), 'cleanup-tmpdir': ("Cleanup tmp dir after successful run.", None, 'store_true', True), 'cpp-headers-search-path': ("Search path used by EasyBuild to inject include directories", 'choice', - 'store', DEFAULT_CPP_HEADER_SEARCH_PATH, [*CPP_HEADER_SEARCH_PATHS]), + 'store', DEFAULT_CPP_HEADERS_SEARCH_PATH, [*CPP_HEADERS_SEARCH_PATHS]), 'color': ("Colorize output", 'choice', 'store', fancylogger.Colorize.AUTO, fancylogger.Colorize, {'metavar': 'WHEN'}), 'consider-archived-easyconfigs': ("Also consider archived easyconfigs", None, 'store_true', False), diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 73b78dd73a..87d5118ca0 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -97,15 +97,15 @@ ] # modes to handle CPP header search paths # see: https://gcc.gnu.org/onlinedocs/cpp/Environment-Variables.html -CPP_HEADER_SEARCH_PATH_FLAGS = "CPPFLAGS" -CPP_HEADER_SEARCH_PATH_CPATH = "CPATH" -CPP_HEADER_SEARCH_PATH_INCLUDE = "INCLUDE_PATHS" -CPP_HEADER_SEARCH_PATHS = { - CPP_HEADER_SEARCH_PATH_FLAGS: ["CPPFLAGS"], - CPP_HEADER_SEARCH_PATH_CPATH: ["CPATH"], - CPP_HEADER_SEARCH_PATH_INCLUDE: ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"], +CPP_HEADERS_SEARCH_PATH_FLAGS = "CPPFLAGS" +CPP_HEADERS_SEARCH_PATH_CPATH = "CPATH" +CPP_HEADERS_SEARCH_PATH_INCLUDE = "INCLUDE_PATHS" +CPP_HEADERS_SEARCH_PATHS = { + CPP_HEADERS_SEARCH_PATH_FLAGS: ["CPPFLAGS"], + CPP_HEADERS_SEARCH_PATH_CPATH: ["CPATH"], + CPP_HEADERS_SEARCH_PATH_INCLUDE: ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"], } -DEFAULT_CPP_HEADER_SEARCH_PATH = CPP_HEADER_SEARCH_PATH_FLAGS +DEFAULT_CPP_HEADERS_SEARCH_PATH = CPP_HEADERS_SEARCH_PATH_FLAGS def is_system_toolchain(tc_name): @@ -1092,15 +1092,15 @@ def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None): # mode of operation is defined by cpp-headers-search-path option # toolchain option has precedence over build option - cpp_headers_mode = DEFAULT_CPP_HEADER_SEARCH_PATH + cpp_headers_mode = DEFAULT_CPP_HEADERS_SEARCH_PATH tc_opt = self.options.option("cpp-headers-search-path") - if tc_opt in CPP_HEADER_SEARCH_PATHS: + if tc_opt in CPP_HEADERS_SEARCH_PATHS: self.log.debug("cpp-headers-search-path set by toolchain option: %s", cpp_headers_mode) cpp_headers_mode = tc_opt elif tc_opt is not None: raise EasyBuildError( "Unknown value selected for toolchain option cpp-headers-search-path. Choose one of: %s", - ", ".join(CPP_HEADER_SEARCH_PATHS) + ", ".join(CPP_HEADERS_SEARCH_PATHS) ) else: build_opt = build_option("cpp_headers_search_path") @@ -1108,7 +1108,7 @@ def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None): cpp_headers_mode = build_opt self.log.debug("cpp-headers-search-path set by build option: %s", cpp_headers_mode) - for env_var in CPP_HEADER_SEARCH_PATHS[cpp_headers_mode]: + for env_var in CPP_HEADERS_SEARCH_PATHS[cpp_headers_mode]: self.log.debug("Adding header paths to toolchain variable '%s': %s", env_var, dep_root) self.variables.append_subdirs(env_var, dep_root, subdirs=header_dirs) From 079f202e7465f22a62b26056eecc9adaf3520f2a Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 18 Sep 2024 16:13:24 +0200 Subject: [PATCH 08/33] simplify parsing of cpp-headers-search-path option values --- easybuild/tools/toolchain/toolchain.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 87d5118ca0..66820d76ba 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1094,19 +1094,20 @@ def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None): # toolchain option has precedence over build option cpp_headers_mode = DEFAULT_CPP_HEADERS_SEARCH_PATH tc_opt = self.options.option("cpp-headers-search-path") - if tc_opt in CPP_HEADERS_SEARCH_PATHS: + if tc_opt is not None: self.log.debug("cpp-headers-search-path set by toolchain option: %s", cpp_headers_mode) cpp_headers_mode = tc_opt - elif tc_opt is not None: - raise EasyBuildError( - "Unknown value selected for toolchain option cpp-headers-search-path. Choose one of: %s", - ", ".join(CPP_HEADERS_SEARCH_PATHS) - ) else: build_opt = build_option("cpp_headers_search_path") if build_opt is not None: - cpp_headers_mode = build_opt self.log.debug("cpp-headers-search-path set by build option: %s", cpp_headers_mode) + cpp_headers_mode = build_opt + + if cpp_headers_mode not in CPP_HEADERS_SEARCH_PATHS: + raise EasyBuildError( + "Unknown value selected for option cpp-headers-search-path. Choose one of: %s", + ", ".join(CPP_HEADERS_SEARCH_PATHS) + ) for env_var in CPP_HEADERS_SEARCH_PATHS[cpp_headers_mode]: self.log.debug("Adding header paths to toolchain variable '%s': %s", env_var, dep_root) From 349386179185211bb273278dc06274b02eceb820 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 18 Sep 2024 16:40:53 +0200 Subject: [PATCH 09/33] simplify FCC toolchain by using inheritance of new Toolchain._add_dependency_cpp_headers method --- easybuild/toolchains/fcc.py | 41 ++++++------------------------------- 1 file changed, 6 insertions(+), 35 deletions(-) diff --git a/easybuild/toolchains/fcc.py b/easybuild/toolchains/fcc.py index 038b7540c3..0da1fc149f 100644 --- a/easybuild/toolchains/fcc.py +++ b/easybuild/toolchains/fcc.py @@ -40,39 +40,10 @@ class FCC(FujitsuCompiler): OPTIONAL = False # override in order to add an exception for the Fujitsu lang/tcsds module - def _add_dependency_variables(self, names=None, cpp=None, ld=None): - """ Add LDFLAGS and CPPFLAGS to the self.variables based on the dependencies - names should be a list of strings containing the name of the dependency + def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None): """ - cpp_paths = ['include'] - ld_paths = ['lib64', 'lib'] - - if cpp is not None: - for p in cpp: - if p not in cpp_paths: - cpp_paths.append(p) - if ld is not None: - for p in ld: - if p not in ld_paths: - ld_paths.append(p) - - if not names: - deps = self.dependencies - else: - deps = [{'name': name} for name in names if name is not None] - - # collect software install prefixes for dependencies - roots = [] - for dep in deps: - if dep.get('external_module', False): - # for software names provided via external modules, install prefix may be unknown - names = dep['external_module_metadata'].get('name', []) - roots.extend([root for root in self.get_software_root(names) if root is not None]) - else: - roots.extend(self.get_software_root(dep['name'])) - - for root in roots: - # skip Fujitsu's 'lang/tcsds' module, including the top level include breaks vectorization in clang mode - if 'tcsds' not in root: - self.variables.append_subdirs("CPPFLAGS", root, subdirs=cpp_paths) - self.variables.append_subdirs("LDFLAGS", root, subdirs=ld_paths) + Append prepocessor paths for given dependency root directory + """ + # skip Fujitsu's 'lang/tcsds' module, including the top level include breaks vectorization in clang mode + if "tcsds" not in dep_root: + super()._add_dependency_cpp_headers(dep_root, extra_dirs=extra_dirs) From 3b3e9d1f6dbc297c610667e696378c87c4d5489e Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 18 Sep 2024 16:55:35 +0200 Subject: [PATCH 10/33] replace hardcoded variable settings in ACML with new Toolchain._add_dependency_liker_paths and Toolchain._add_dependency_cpp_headers methods --- easybuild/toolchains/linalg/acml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/toolchains/linalg/acml.py b/easybuild/toolchains/linalg/acml.py index 1470b1f360..e38424726b 100644 --- a/easybuild/toolchains/linalg/acml.py +++ b/easybuild/toolchains/linalg/acml.py @@ -77,9 +77,9 @@ def _set_blas_variables(self): for root in self.get_software_root(self.BLAS_MODULE_NAME): subdirs = self.ACML_SUBDIRS_MAP[self.COMPILER_FAMILY] self.BLAS_LIB_DIR = [os.path.join(x, 'lib') for x in subdirs] - self.variables.append_exists('LDFLAGS', root, self.BLAS_LIB_DIR, append_all=True) + self._add_dependency_linker_paths(root, extra_dirs=self.BLAS_LIB_DIR) incdirs = [os.path.join(x, 'include') for x in subdirs] - self.variables.append_exists('CPPFLAGS', root, incdirs, append_all=True) + self._add_dependency_cpp_headers(root, extra_dirs=incdirs) except Exception: raise EasyBuildError("_set_blas_variables: ACML set LDFLAGS/CPPFLAGS unknown entry in ACML_SUBDIRS_MAP " "with compiler family %s", self.COMPILER_FAMILY) From f0190f7922ae8a2c112fbe76d78e4a7a87998f83 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 19 Sep 2024 01:44:28 +0200 Subject: [PATCH 11/33] add test_cpp_headers_search_path to test.framework.toolchain --- test/framework/toolchain.py | 43 +++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 2e8c7893b1..abf3176fe8 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -955,6 +955,49 @@ def test_precision_flags(self): self.modtool.purge() + def test_cpp_headers_search_path(self): + """Test functionality behind cpp-headers-search-path option""" + cpp_headers_mode = { + "CPPFLAGS": ["CPPFLAGS"], + "CPATH": ["CPATH"], + "INCLUDE_PATHS": ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"], + } + # test without toolchain option + for build_opt in cpp_headers_mode: + init_config(build_options={"cpp_headers_search_path": build_opt, "silent": True}) + tc = self.get_toolchain("foss", version="2018a") + with self.mocked_stdout_stderr(): + tc.prepare() + for env_var in cpp_headers_mode[build_opt]: + assert_fail_msg = ( + f"Variable {env_var} required by cpp-headers-search-path build option '{build_opt}' " + "not found in toolchain environment" + ) + self.assertIn(env_var, [*tc.variables], assert_fail_msg) + self.modtool.purge() + # test with toolchain option + for build_opt in cpp_headers_mode: + init_config(build_options={"cpp_headers_search_path": build_opt, "silent": True}) + for tc_opt in cpp_headers_mode: + tc = self.get_toolchain("foss", version="2018a") + tc.set_options({"cpp-headers-search-path": tc_opt}) + with self.mocked_stdout_stderr(): + tc.prepare() + for env_var in cpp_headers_mode[tc_opt]: + assert_fail_msg = ( + f"Variable {env_var} required by cpp-headers-search-path toolchain option '{tc_opt}' " + "not found in toolchain environment" + ) + self.assertIn(env_var, [*tc.variables], assert_fail_msg) + self.modtool.purge() + # test wrong toolchain option + tc = self.get_toolchain("foss", version="2018a") + tc.set_options({"cpp-headers-search-path": "WRONG_MODE"}) + with self.mocked_stdout_stderr(): + error_pattern = "Unknown value selected for option cpp-headers-search-path" + self.assertErrorRegex(EasyBuildError, error_pattern, tc.prepare) + self.modtool.purge() + def test_cgoolf_toolchain(self): """Test for cgoolf toolchain.""" tc = self.get_toolchain("cgoolf", version="1.1.6") From 6471f17ca4f3351fbe56756e90bbc1ed84f1e601 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 24 Sep 2024 23:49:07 +0200 Subject: [PATCH 12/33] rename cpp-headers-search-path option to search-path-cpp-headers --- easybuild/tools/config.py | 2 +- easybuild/tools/options.py | 6 ++--- easybuild/tools/toolchain/compiler.py | 2 +- easybuild/tools/toolchain/toolchain.py | 36 +++++++++++++------------- test/framework/toolchain.py | 18 ++++++------- 5 files changed, 32 insertions(+), 32 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 8f07f298b4..0d04becb49 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -206,7 +206,6 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'backup_modules', 'banned_linked_shared_libs', 'checksum_priority', - 'cpp_headers_search_path', 'container_config', 'container_image_format', 'container_image_name', @@ -262,6 +261,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'rpath_filter', 'rpath_override_dirs', 'required_linked_shared_libs', + 'search_path_cpp_headers', 'skip', 'software_commit', 'stop', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 515bc535b1..aee8d9a593 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -101,7 +101,7 @@ from easybuild.tools.run import run_shell_cmd from easybuild.tools.package.utilities import avail_package_naming_schemes from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, OPTARCH_MAP_CHAR, OPTARCH_SEP, Compiler -from easybuild.tools.toolchain.toolchain import CPP_HEADERS_SEARCH_PATHS, DEFAULT_CPP_HEADERS_SEARCH_PATH +from easybuild.tools.toolchain.toolchain import SEARCH_PATH_CPP_HEADERS, DEFAULT_SEARCH_PATH_CPP_HEADERS from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME from easybuild.tools.repository.repository import avail_repositories from easybuild.tools.systemtools import DARWIN, UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family @@ -376,8 +376,6 @@ def override_options(self): 'choice', 'store_or_None', DEFAULT_CHECKSUM_PRIORITY, CHECKSUM_PRIORITY_CHOICES), 'cleanup-builddir': ("Cleanup build dir after successful installation.", None, 'store_true', True), 'cleanup-tmpdir': ("Cleanup tmp dir after successful run.", None, 'store_true', True), - 'cpp-headers-search-path': ("Search path used by EasyBuild to inject include directories", 'choice', - 'store', DEFAULT_CPP_HEADERS_SEARCH_PATH, [*CPP_HEADERS_SEARCH_PATHS]), 'color': ("Colorize output", 'choice', 'store', fancylogger.Colorize.AUTO, fancylogger.Colorize, {'metavar': 'WHEN'}), 'consider-archived-easyconfigs': ("Also consider archived easyconfigs", None, 'store_true', False), @@ -626,6 +624,8 @@ def config_options(self): "(is passed as list of arguments to create the repository instance). " "For more info, use --avail-repositories."), 'strlist', 'store', self.default_repositorypath), + 'search-path-cpp-headers': ("Search path used at build time for include directories", 'choice', + 'store', DEFAULT_SEARCH_PATH_CPP_HEADERS, [*SEARCH_PATH_CPP_HEADERS]), 'sourcepath': ("Path(s) to where sources should be downloaded (string, colon-separated)", None, 'store', mk_full_default_path('sourcepath')), 'subdir-modules': ("Installpath subdir for modules", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_modules']), diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index cf67003570..7f768a134a 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -93,7 +93,7 @@ class Compiler(Toolchain): 'vectorize': (None, "Enable compiler auto-vectorization, default except for noopt and lowopt"), 'packed-linker-options': (False, "Pack the linker options as comma separated list"), # ScaLAPACK mainly 'rpath': (True, "Use RPATH wrappers when --rpath is enabled in EasyBuild configuration"), - 'cpp-headers-search-path': (None, "Search path used by EasyBuild to inject include directories"), + 'search-path-cpp-headers': (None, "Search path used at build time for include directories"), 'extra_cflags': (None, "Specify extra CFLAGS options."), 'extra_cxxflags': (None, "Specify extra CXXFLAGS options."), 'extra_fflags': (None, "Specify extra FFLAGS options."), diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 66820d76ba..7ab27c9f9d 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -97,15 +97,15 @@ ] # modes to handle CPP header search paths # see: https://gcc.gnu.org/onlinedocs/cpp/Environment-Variables.html -CPP_HEADERS_SEARCH_PATH_FLAGS = "CPPFLAGS" -CPP_HEADERS_SEARCH_PATH_CPATH = "CPATH" -CPP_HEADERS_SEARCH_PATH_INCLUDE = "INCLUDE_PATHS" -CPP_HEADERS_SEARCH_PATHS = { - CPP_HEADERS_SEARCH_PATH_FLAGS: ["CPPFLAGS"], - CPP_HEADERS_SEARCH_PATH_CPATH: ["CPATH"], - CPP_HEADERS_SEARCH_PATH_INCLUDE: ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"], +SEARCH_PATH_CPP_HEADERS_FLAGS = "CPPFLAGS" +SEARCH_PATH_CPP_HEADERS_CPATH = "CPATH" +SEARCH_PATH_CPP_HEADERS_INCLUDE = "INCLUDE_PATHS" +SEARCH_PATH_CPP_HEADERS = { + SEARCH_PATH_CPP_HEADERS_FLAGS: ["CPPFLAGS"], + SEARCH_PATH_CPP_HEADERS_CPATH: ["CPATH"], + SEARCH_PATH_CPP_HEADERS_INCLUDE: ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"], } -DEFAULT_CPP_HEADERS_SEARCH_PATH = CPP_HEADERS_SEARCH_PATH_FLAGS +DEFAULT_SEARCH_PATH_CPP_HEADERS = SEARCH_PATH_CPP_HEADERS_FLAGS def is_system_toolchain(tc_name): @@ -1090,26 +1090,26 @@ def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None): header_dirs.extend(extra_dirs) header_dirs = set(header_dirs) # remove duplicates - # mode of operation is defined by cpp-headers-search-path option + # mode of operation is defined by search-path-cpp-headers option # toolchain option has precedence over build option - cpp_headers_mode = DEFAULT_CPP_HEADERS_SEARCH_PATH - tc_opt = self.options.option("cpp-headers-search-path") + cpp_headers_mode = DEFAULT_SEARCH_PATH_CPP_HEADERS + tc_opt = self.options.option("search-path-cpp-headers") if tc_opt is not None: - self.log.debug("cpp-headers-search-path set by toolchain option: %s", cpp_headers_mode) + self.log.debug("search-path-cpp-headers set by toolchain option: %s", cpp_headers_mode) cpp_headers_mode = tc_opt else: - build_opt = build_option("cpp_headers_search_path") + build_opt = build_option("search_path_cpp_headers") if build_opt is not None: - self.log.debug("cpp-headers-search-path set by build option: %s", cpp_headers_mode) + self.log.debug("search-path-cpp-headers set by build option: %s", cpp_headers_mode) cpp_headers_mode = build_opt - if cpp_headers_mode not in CPP_HEADERS_SEARCH_PATHS: + if cpp_headers_mode not in SEARCH_PATH_CPP_HEADERS: raise EasyBuildError( - "Unknown value selected for option cpp-headers-search-path. Choose one of: %s", - ", ".join(CPP_HEADERS_SEARCH_PATHS) + "Unknown value selected for option search-path-cpp-headers. Choose one of: %s", + ", ".join(SEARCH_PATH_CPP_HEADERS) ) - for env_var in CPP_HEADERS_SEARCH_PATHS[cpp_headers_mode]: + for env_var in SEARCH_PATH_CPP_HEADERS[cpp_headers_mode]: self.log.debug("Adding header paths to toolchain variable '%s': %s", env_var, dep_root) self.variables.append_subdirs(env_var, dep_root, subdirs=header_dirs) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index abf3176fe8..f28814556e 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -955,8 +955,8 @@ def test_precision_flags(self): self.modtool.purge() - def test_cpp_headers_search_path(self): - """Test functionality behind cpp-headers-search-path option""" + def test_search_path_cpp_headers(self): + """Test functionality behind search-path-cpp-headers option""" cpp_headers_mode = { "CPPFLAGS": ["CPPFLAGS"], "CPATH": ["CPATH"], @@ -964,37 +964,37 @@ def test_cpp_headers_search_path(self): } # test without toolchain option for build_opt in cpp_headers_mode: - init_config(build_options={"cpp_headers_search_path": build_opt, "silent": True}) + init_config(build_options={"search_path_cpp_headers": build_opt, "silent": True}) tc = self.get_toolchain("foss", version="2018a") with self.mocked_stdout_stderr(): tc.prepare() for env_var in cpp_headers_mode[build_opt]: assert_fail_msg = ( - f"Variable {env_var} required by cpp-headers-search-path build option '{build_opt}' " + f"Variable {env_var} required by search-path-cpp-headers build option '{build_opt}' " "not found in toolchain environment" ) self.assertIn(env_var, [*tc.variables], assert_fail_msg) self.modtool.purge() # test with toolchain option for build_opt in cpp_headers_mode: - init_config(build_options={"cpp_headers_search_path": build_opt, "silent": True}) + init_config(build_options={"search_path_cpp_headers": build_opt, "silent": True}) for tc_opt in cpp_headers_mode: tc = self.get_toolchain("foss", version="2018a") - tc.set_options({"cpp-headers-search-path": tc_opt}) + tc.set_options({"search-path-cpp-headers": tc_opt}) with self.mocked_stdout_stderr(): tc.prepare() for env_var in cpp_headers_mode[tc_opt]: assert_fail_msg = ( - f"Variable {env_var} required by cpp-headers-search-path toolchain option '{tc_opt}' " + f"Variable {env_var} required by search-path-cpp-headers toolchain option '{tc_opt}' " "not found in toolchain environment" ) self.assertIn(env_var, [*tc.variables], assert_fail_msg) self.modtool.purge() # test wrong toolchain option tc = self.get_toolchain("foss", version="2018a") - tc.set_options({"cpp-headers-search-path": "WRONG_MODE"}) + tc.set_options({"search-path-cpp-headers": "WRONG_MODE"}) with self.mocked_stdout_stderr(): - error_pattern = "Unknown value selected for option cpp-headers-search-path" + error_pattern = "Unknown value selected for option search-path-cpp-headers" self.assertErrorRegex(EasyBuildError, error_pattern, tc.prepare) self.modtool.purge() From 45514efff4ee697830c3cfdf1349d8bdc446ba78 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 25 Sep 2024 00:19:44 +0200 Subject: [PATCH 13/33] set default value of toolchain option search-path-cpp-headers to False --- easybuild/tools/toolchain/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 7f768a134a..c6cce8ddf9 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -93,7 +93,7 @@ class Compiler(Toolchain): 'vectorize': (None, "Enable compiler auto-vectorization, default except for noopt and lowopt"), 'packed-linker-options': (False, "Pack the linker options as comma separated list"), # ScaLAPACK mainly 'rpath': (True, "Use RPATH wrappers when --rpath is enabled in EasyBuild configuration"), - 'search-path-cpp-headers': (None, "Search path used at build time for include directories"), + 'search-path-cpp-headers': (False, "Search path used at build time for include directories"), 'extra_cflags': (None, "Specify extra CFLAGS options."), 'extra_cxxflags': (None, "Specify extra CXXFLAGS options."), 'extra_fflags': (None, "Specify extra FFLAGS options."), From 96b020ebee90180446973136df1c12b12cadab40 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 25 Sep 2024 00:21:01 +0200 Subject: [PATCH 14/33] simplify search-path-cpp-headers option detection as its build option cannot be None --- easybuild/tools/toolchain/toolchain.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 7ab27c9f9d..04bdbf9b65 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1092,21 +1092,19 @@ def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None): # mode of operation is defined by search-path-cpp-headers option # toolchain option has precedence over build option - cpp_headers_mode = DEFAULT_SEARCH_PATH_CPP_HEADERS + cpp_headers_mode = SEARCH_PATH_CPP_HEADERS tc_opt = self.options.option("search-path-cpp-headers") - if tc_opt is not None: - self.log.debug("search-path-cpp-headers set by toolchain option: %s", cpp_headers_mode) + if tc_opt is not False: cpp_headers_mode = tc_opt + self.log.debug("search-path-cpp-headers set by toolchain option: %s", cpp_headers_mode) else: - build_opt = build_option("search_path_cpp_headers") - if build_opt is not None: - self.log.debug("search-path-cpp-headers set by build option: %s", cpp_headers_mode) - cpp_headers_mode = build_opt + cpp_headers_mode = build_option("search_path_cpp_headers") + self.log.debug("search-path-cpp-headers set by build option: %s", cpp_headers_mode) if cpp_headers_mode not in SEARCH_PATH_CPP_HEADERS: raise EasyBuildError( - "Unknown value selected for option search-path-cpp-headers. Choose one of: %s", - ", ".join(SEARCH_PATH_CPP_HEADERS) + "Unknown value selected for option search-path-cpp-headers: %s. Choose one of: %s", + cpp_headers_mode, ", ".join(SEARCH_PATH_CPP_HEADERS) ) for env_var in SEARCH_PATH_CPP_HEADERS[cpp_headers_mode]: From 35db2f7ce95500e208e895df3840117db769af08 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 25 Sep 2024 11:17:56 +0200 Subject: [PATCH 15/33] toolchain option search_path_ccp_headers can be None in unit tests --- easybuild/tools/toolchain/toolchain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 04bdbf9b65..0921dc6d6c 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1092,9 +1092,9 @@ def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None): # mode of operation is defined by search-path-cpp-headers option # toolchain option has precedence over build option - cpp_headers_mode = SEARCH_PATH_CPP_HEADERS + cpp_headers_mode = DEFAULT_SEARCH_PATH_CPP_HEADERS tc_opt = self.options.option("search-path-cpp-headers") - if tc_opt is not False: + if tc_opt: cpp_headers_mode = tc_opt self.log.debug("search-path-cpp-headers set by toolchain option: %s", cpp_headers_mode) else: From 8cbb705f81f3aa9fe7e5adbee1e0e486e575f7e8 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 25 Sep 2024 11:38:30 +0200 Subject: [PATCH 16/33] build option search_path_ccp_headers can be None in unit tests --- easybuild/tools/toolchain/toolchain.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 0921dc6d6c..22dda80fd0 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1094,11 +1094,12 @@ def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None): # toolchain option has precedence over build option cpp_headers_mode = DEFAULT_SEARCH_PATH_CPP_HEADERS tc_opt = self.options.option("search-path-cpp-headers") + build_opt = build_option("search_path_cpp_headers") if tc_opt: cpp_headers_mode = tc_opt self.log.debug("search-path-cpp-headers set by toolchain option: %s", cpp_headers_mode) - else: - cpp_headers_mode = build_option("search_path_cpp_headers") + elif build_opt: + cpp_headers_mode = build_opt self.log.debug("search-path-cpp-headers set by build option: %s", cpp_headers_mode) if cpp_headers_mode not in SEARCH_PATH_CPP_HEADERS: From 3daac79016032850a923190f2c462b4271dcc779 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 26 Sep 2024 10:45:30 +0200 Subject: [PATCH 17/33] avoid list conversion in test_search_path_cpp_headers Co-authored-by: Alexander Grund --- test/framework/toolchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index f28814556e..72759a11a3 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -973,7 +973,7 @@ def test_search_path_cpp_headers(self): f"Variable {env_var} required by search-path-cpp-headers build option '{build_opt}' " "not found in toolchain environment" ) - self.assertIn(env_var, [*tc.variables], assert_fail_msg) + self.assertIn(env_var, tc.variables, assert_fail_msg) self.modtool.purge() # test with toolchain option for build_opt in cpp_headers_mode: From a110459db2bdd5a07e1c85b102bfa7df7978e6ac Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 26 Sep 2024 10:46:20 +0200 Subject: [PATCH 18/33] use None as default of toolchain option search-path-cpp-headers Co-authored-by: Alexander Grund --- easybuild/tools/toolchain/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index c6cce8ddf9..7f768a134a 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -93,7 +93,7 @@ class Compiler(Toolchain): 'vectorize': (None, "Enable compiler auto-vectorization, default except for noopt and lowopt"), 'packed-linker-options': (False, "Pack the linker options as comma separated list"), # ScaLAPACK mainly 'rpath': (True, "Use RPATH wrappers when --rpath is enabled in EasyBuild configuration"), - 'search-path-cpp-headers': (False, "Search path used at build time for include directories"), + 'search-path-cpp-headers': (None, "Search path used at build time for include directories"), 'extra_cflags': (None, "Specify extra CFLAGS options."), 'extra_cxxflags': (None, "Specify extra CXXFLAGS options."), 'extra_fflags': (None, "Specify extra FFLAGS options."), From 324c79c1cbef9ba200e974d5af2ff12b579f47b2 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 26 Sep 2024 10:47:45 +0200 Subject: [PATCH 19/33] avoid warning about search-path-cpp-headers returning none --- easybuild/tools/toolchain/toolchain.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 22dda80fd0..c3b2534006 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1093,12 +1093,11 @@ def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None): # mode of operation is defined by search-path-cpp-headers option # toolchain option has precedence over build option cpp_headers_mode = DEFAULT_SEARCH_PATH_CPP_HEADERS - tc_opt = self.options.option("search-path-cpp-headers") build_opt = build_option("search_path_cpp_headers") - if tc_opt: - cpp_headers_mode = tc_opt + if self.options.get("search-path-cpp-headers") is not None: + cpp_headers_mode = self.options.option("search-path-cpp-headers") self.log.debug("search-path-cpp-headers set by toolchain option: %s", cpp_headers_mode) - elif build_opt: + elif build_opt is not None: cpp_headers_mode = build_opt self.log.debug("search-path-cpp-headers set by build option: %s", cpp_headers_mode) From 1344b974550fe8248b5e431914ae5533c42b80d5 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 26 Sep 2024 10:57:20 +0200 Subject: [PATCH 20/33] fix typo in comment of toolchain.prepare method --- easybuild/tools/toolchain/toolchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index c3b2534006..680e61fc14 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -861,7 +861,7 @@ def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True, else: self.log.debug("prepare: set additional variables onlymod=%s", onlymod) - # add linker and preprocessor paths to dependencies to self.vars + # add linker and preprocessor paths of dependencies to self.vars self._add_dependency_variables() self.generate_vars() self._setenv_variables(onlymod, verbose=not silent) From 7e3b0701dcfd99427321931407798d266f6753cd Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 26 Sep 2024 11:00:01 +0200 Subject: [PATCH 21/33] fix typo in docstring of toolchain._add_dependency_variables method --- easybuild/tools/toolchain/toolchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 680e61fc14..6564c805fc 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1061,7 +1061,7 @@ def handle_sysroot(self): def _add_dependency_variables(self, names=None, cpp=None, ld=None): """ - Add linker and preprocessor paths to dependencies to self.variables + Add linker and preprocessor paths of dependencies to self.variables :names: list of strings containing the name of the dependency """ # collect dependencies From 77a65ad7f8199e9bf261878d135846e393a1e0dd Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 26 Sep 2024 11:17:55 +0200 Subject: [PATCH 22/33] remove duplicates with tools.utilities.nub instead of a set to preserve order --- easybuild/tools/toolchain/toolchain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 6564c805fc..cf6d8b0752 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1088,7 +1088,7 @@ def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None): header_dirs = ["include"] if isinstance(extra_dirs, (tuple, list)): header_dirs.extend(extra_dirs) - header_dirs = set(header_dirs) # remove duplicates + header_dirs = nub(header_dirs) # remove duplicates # mode of operation is defined by search-path-cpp-headers option # toolchain option has precedence over build option @@ -1118,7 +1118,7 @@ def _add_dependency_linker_paths(self, dep_root, extra_dirs=None): lib_dirs = ["lib64", "lib"] if isinstance(extra_dirs, (tuple, list)): lib_dirs.extend(extra_dirs) - lib_dirs = set(lib_dirs) # remove duplicates + lib_dirs = nub(lib_dirs) # remove duplicates env_var = "LDFLAGS" self.log.debug("Adding lib paths to toolchain variable '%s': %s", env_var, dep_root) From 80e5825ab493b1cbac2aff8940482e797c2a6105 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 26 Sep 2024 11:24:44 +0200 Subject: [PATCH 23/33] avoid list conversion in test_search_path_cpp_headers Co-authored-by: Alexander Grund --- test/framework/toolchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 72759a11a3..779b0ed182 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -988,7 +988,7 @@ def test_search_path_cpp_headers(self): f"Variable {env_var} required by search-path-cpp-headers toolchain option '{tc_opt}' " "not found in toolchain environment" ) - self.assertIn(env_var, [*tc.variables], assert_fail_msg) + self.assertIn(env_var, tc.variables, assert_fail_msg) self.modtool.purge() # test wrong toolchain option tc = self.get_toolchain("foss", version="2018a") From cd01b6564498618f689937af9bdd7904e56a1e73 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 26 Sep 2024 16:19:29 +0200 Subject: [PATCH 24/33] handle extra_dirs passed as string and throw error for non-iterables --- easybuild/tools/toolchain/toolchain.py | 31 ++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index cf6d8b0752..3b70365e0f 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1086,9 +1086,19 @@ def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None): Append prepocessor paths for given dependency root directory """ header_dirs = ["include"] - if isinstance(extra_dirs, (tuple, list)): + + if extra_dirs is None: + extra_dirs = () + elif extra_dirs and isinstance(extra_dirs, (str,)): + extra_dirs = [extra_dirs] + + try: header_dirs.extend(extra_dirs) - header_dirs = nub(header_dirs) # remove duplicates + except TypeError as err: + err_msg = f"_add_dependency_cpp_headers: given extra_dirs is not iterable: {extra_dirs}" + raise EasyBuildError(err_msg) from err + else: + header_dirs = nub(header_dirs) # remove duplicates # mode of operation is defined by search-path-cpp-headers option # toolchain option has precedence over build option @@ -1116,14 +1126,27 @@ def _add_dependency_linker_paths(self, dep_root, extra_dirs=None): Append linker paths for given dependency root directory """ lib_dirs = ["lib64", "lib"] - if isinstance(extra_dirs, (tuple, list)): + + if extra_dirs is None: + extra_dirs = () + elif extra_dirs and isinstance(extra_dirs, (str,)): + extra_dirs = [extra_dirs] + + try: lib_dirs.extend(extra_dirs) - lib_dirs = nub(lib_dirs) # remove duplicates + except TypeError as err: + err_msg = f"_add_dependency_linker_paths: given extra_dirs is not iterable: {extra_dirs}" + raise EasyBuildError(err_msg) from err + else: + lib_dirs = nub(lib_dirs) # remove duplicates env_var = "LDFLAGS" self.log.debug("Adding lib paths to toolchain variable '%s': %s", env_var, dep_root) self.variables.append_subdirs(env_var, dep_root, subdirs=lib_dirs) + def _unique_ordered_list_append(self, base, extra_dirs): + """Append list of dirs to another list of dirs keeping order and without duplicates""" + def _setenv_variables(self, donotset=None, verbose=True): """Actually set the environment variables""" From d12251faad00a4b66b52bb2d7fe959a71f8323d9 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 30 Sep 2024 10:55:36 +0200 Subject: [PATCH 25/33] import all of easybuild.tools.utilities with an alias in its test suite --- test/framework/utilities_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py index c49b73ba0b..0e0c7642b6 100644 --- a/test/framework/utilities_test.py +++ b/test/framework/utilities_test.py @@ -36,10 +36,10 @@ from datetime import datetime from unittest import TextTestRunner +import easybuild.tools.utilities as tu from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.utilities import time2str, natural_keys class UtilitiesTest(EnhancedTestCase): @@ -77,10 +77,10 @@ def test_time2str(self): (datetime(2019, 8, 5, 20, 39, 44), "159 hours 25 mins 21 secs"), ] for end, expected in test_cases: - self.assertEqual(time2str(end - start), expected) + self.assertEqual(tu.time2str(end - start), expected) error_pattern = "Incorrect value type provided to time2str, should be datetime.timedelta: <.* 'int'>" - self.assertErrorRegex(EasyBuildError, error_pattern, time2str, 123) + self.assertErrorRegex(EasyBuildError, error_pattern, tu.time2str, 123) def test_natural_keys(self): """Test the natural_keys function""" @@ -98,7 +98,7 @@ def test_natural_keys(self): ] shuffled_items = sorted_items[:] random.shuffle(shuffled_items) - shuffled_items.sort(key=natural_keys) + shuffled_items.sort(key=tu.natural_keys) self.assertEqual(shuffled_items, sorted_items) def test_LooseVersion(self): From b714e65e124949ab64b177c8e57c1d4f55c4cf7c Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 30 Sep 2024 10:56:03 +0200 Subject: [PATCH 26/33] add new emthod unique_ordered_append to easybuild.tools.utilities --- easybuild/tools/utilities.py | 19 +++++++++++++++++++ test/framework/utilities_test.py | 17 +++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index fe254ea4d2..1b4aa05033 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -223,6 +223,25 @@ def nub(list_): return [x for x in list_ if x not in seen and not seen_add(x)] +def unique_ordered_append(base, extra): + """Append elements of extra list to another base list keeping order and without duplicates""" + if extra and isinstance(extra, str): + extra = [extra] + + try: + base.extend(extra) + except TypeError as err: + err_msg = f"_unique_ordered_list_append: given extra list is not iterable: {extra}" + raise EasyBuildError(err_msg) from err + except AttributeError as err: + err_msg = f"_unique_ordered_list_append: given base cannot be extended: {base}" + raise EasyBuildError(err_msg) from err + else: + base = nub(base) # remove duplicates + + return base + + def get_class_for(modulepath, class_name): """ Get class for a given Python class name and Python module path. diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py index 0e0c7642b6..ac929b6c84 100644 --- a/test/framework/utilities_test.py +++ b/test/framework/utilities_test.py @@ -191,6 +191,23 @@ def test_LooseVersion(self): self.assertEqual(LooseVersion('2.a').version, [2, 'a']) self.assertEqual(LooseVersion('2.a5').version, [2, 'a', 5]) + def test_unique_ordered_append(self): + """Test unique_ordered_list_append method""" + base = ["potato", "tomato", "orange"] + + reference = ["potato", "tomato", "orange", "apple"] + self.assertEqual(tu.unique_ordered_append(base, ["apple"]), reference) + self.assertEqual(tu.unique_ordered_append(base, ["apple", "apple"]), reference) + self.assertEqual(tu.unique_ordered_append(base, "apple"), reference) + self.assertNotEqual(tu.unique_ordered_append(base, "apple"), sorted(reference)) + + error_pattern = "given extra list is not iterable" + self.assertErrorRegex(EasyBuildError, error_pattern, tu.unique_ordered_append, base, 0) + + base = "potato" + error_pattern = "given base cannot be extended" + self.assertErrorRegex(EasyBuildError, error_pattern, tu.unique_ordered_append, base, reference) + def suite(): """ return all the tests in this file """ From e1ec9d95baada55b533d27a2d9248c98dec4ba70 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 30 Sep 2024 11:02:49 +0200 Subject: [PATCH 27/33] simplify handling of search-path-cpp-headers option with tools.utilities.unique_ordered_append --- easybuild/tools/toolchain/toolchain.py | 35 ++++++-------------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 3b70365e0f..bf39635624 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -69,7 +69,7 @@ from easybuild.tools.systemtools import LINUX, get_os_type from easybuild.tools.toolchain.options import ToolchainOptions from easybuild.tools.toolchain.toolchainvariables import ToolchainVariables -from easybuild.tools.utilities import nub, trace_msg +from easybuild.tools.utilities import nub, unique_ordered_append, trace_msg _log = fancylogger.getLogger('tools.toolchain', fname=False) @@ -1085,20 +1085,11 @@ def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None): """ Append prepocessor paths for given dependency root directory """ - header_dirs = ["include"] - if extra_dirs is None: extra_dirs = () - elif extra_dirs and isinstance(extra_dirs, (str,)): - extra_dirs = [extra_dirs] - - try: - header_dirs.extend(extra_dirs) - except TypeError as err: - err_msg = f"_add_dependency_cpp_headers: given extra_dirs is not iterable: {extra_dirs}" - raise EasyBuildError(err_msg) from err - else: - header_dirs = nub(header_dirs) # remove duplicates + + header_dirs = ["include"] + header_dirs = unique_ordered_append(header_dirs, extra_dirs) # mode of operation is defined by search-path-cpp-headers option # toolchain option has precedence over build option @@ -1125,28 +1116,16 @@ def _add_dependency_linker_paths(self, dep_root, extra_dirs=None): """ Append linker paths for given dependency root directory """ - lib_dirs = ["lib64", "lib"] - if extra_dirs is None: extra_dirs = () - elif extra_dirs and isinstance(extra_dirs, (str,)): - extra_dirs = [extra_dirs] - - try: - lib_dirs.extend(extra_dirs) - except TypeError as err: - err_msg = f"_add_dependency_linker_paths: given extra_dirs is not iterable: {extra_dirs}" - raise EasyBuildError(err_msg) from err - else: - lib_dirs = nub(lib_dirs) # remove duplicates + + lib_dirs = ["lib64", "lib"] + lib_dirs = unique_ordered_append(lib_dirs, extra_dirs) env_var = "LDFLAGS" self.log.debug("Adding lib paths to toolchain variable '%s': %s", env_var, dep_root) self.variables.append_subdirs(env_var, dep_root, subdirs=lib_dirs) - def _unique_ordered_list_append(self, base, extra_dirs): - """Append list of dirs to another list of dirs keeping order and without duplicates""" - def _setenv_variables(self, donotset=None, verbose=True): """Actually set the environment variables""" From d03ff298dd81ccbfee5af5529bfe90971a400b69 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 2 Oct 2024 13:10:55 +0200 Subject: [PATCH 28/33] simplify error message in unique_ordered_append Co-authored-by: Alexander Grund --- easybuild/tools/utilities.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index 1b4aa05033..ca026e46d4 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -231,8 +231,7 @@ def unique_ordered_append(base, extra): try: base.extend(extra) except TypeError as err: - err_msg = f"_unique_ordered_list_append: given extra list is not iterable: {extra}" - raise EasyBuildError(err_msg) from err + raise EasyBuildError( f"given extra list is not iterable: {extra}") from err except AttributeError as err: err_msg = f"_unique_ordered_list_append: given base cannot be extended: {base}" raise EasyBuildError(err_msg) from err From ce18c9e224f3576afe2961387c5d279f0737ca70 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 2 Oct 2024 13:11:11 +0200 Subject: [PATCH 29/33] simplify error message in unique_ordered_append Co-authored-by: Alexander Grund --- easybuild/tools/utilities.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index ca026e46d4..1b310157bb 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -233,8 +233,7 @@ def unique_ordered_append(base, extra): except TypeError as err: raise EasyBuildError( f"given extra list is not iterable: {extra}") from err except AttributeError as err: - err_msg = f"_unique_ordered_list_append: given base cannot be extended: {base}" - raise EasyBuildError(err_msg) from err + raise EasyBuildError(f"given base cannot be extended: {base}") from err else: base = nub(base) # remove duplicates From df086ae87fd0d561fd3dfbd9dd29b42a4b0908c8 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 2 Oct 2024 13:20:40 +0200 Subject: [PATCH 30/33] simplify else codepath in unique_ordered_append Co-authored-by: Alexander Grund --- easybuild/tools/utilities.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index 1b310157bb..07e84f3854 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -234,10 +234,8 @@ def unique_ordered_append(base, extra): raise EasyBuildError( f"given extra list is not iterable: {extra}") from err except AttributeError as err: raise EasyBuildError(f"given base cannot be extended: {base}") from err - else: - base = nub(base) # remove duplicates - return base + return nub(base) # remove duplicates def get_class_for(modulepath, class_name): From 4d1b56ee64abdd83136bd4d97379df9679cc4885 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 2 Oct 2024 13:23:05 +0200 Subject: [PATCH 31/33] fix codestyle in unique_ordered_append --- easybuild/tools/utilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index 07e84f3854..8ebf79e6b4 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -231,7 +231,7 @@ def unique_ordered_append(base, extra): try: base.extend(extra) except TypeError as err: - raise EasyBuildError( f"given extra list is not iterable: {extra}") from err + raise EasyBuildError(f"given extra list is not iterable: {extra}") from err except AttributeError as err: raise EasyBuildError(f"given base cannot be extended: {base}") from err From c0662ae7e5702f75081c76b3cc866ebb407ac686 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 2 Oct 2024 13:57:04 +0200 Subject: [PATCH 32/33] rename unique_ordered_append to unique_ordered_extend and enforce closer behaviour to extend --- easybuild/tools/toolchain/toolchain.py | 6 +++--- easybuild/tools/utilities.py | 9 +++++---- test/framework/utilities_test.py | 14 +++++++------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index bf39635624..bf1ddd8f1e 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -69,7 +69,7 @@ from easybuild.tools.systemtools import LINUX, get_os_type from easybuild.tools.toolchain.options import ToolchainOptions from easybuild.tools.toolchain.toolchainvariables import ToolchainVariables -from easybuild.tools.utilities import nub, unique_ordered_append, trace_msg +from easybuild.tools.utilities import nub, unique_ordered_extend, trace_msg _log = fancylogger.getLogger('tools.toolchain', fname=False) @@ -1089,7 +1089,7 @@ def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None): extra_dirs = () header_dirs = ["include"] - header_dirs = unique_ordered_append(header_dirs, extra_dirs) + header_dirs = unique_ordered_extend(header_dirs, extra_dirs) # mode of operation is defined by search-path-cpp-headers option # toolchain option has precedence over build option @@ -1120,7 +1120,7 @@ def _add_dependency_linker_paths(self, dep_root, extra_dirs=None): extra_dirs = () lib_dirs = ["lib64", "lib"] - lib_dirs = unique_ordered_append(lib_dirs, extra_dirs) + lib_dirs = unique_ordered_extend(lib_dirs, extra_dirs) env_var = "LDFLAGS" self.log.debug("Adding lib paths to toolchain variable '%s': %s", env_var, dep_root) diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index 8ebf79e6b4..0b7b50475f 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -223,10 +223,11 @@ def nub(list_): return [x for x in list_ if x not in seen and not seen_add(x)] -def unique_ordered_append(base, extra): - """Append elements of extra list to another base list keeping order and without duplicates""" - if extra and isinstance(extra, str): - extra = [extra] +def unique_ordered_extend(base, extra): + """Extend base list with elements of extra list keeping order and without duplicates""" + if isinstance(extra, str): + # avoid strings as they are iterables and generate wrong result without error + raise EasyBuildError(f"given extra list is a string: {extra}") try: base.extend(extra) diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py index ac929b6c84..f1df4bb85f 100644 --- a/test/framework/utilities_test.py +++ b/test/framework/utilities_test.py @@ -191,22 +191,22 @@ def test_LooseVersion(self): self.assertEqual(LooseVersion('2.a').version, [2, 'a']) self.assertEqual(LooseVersion('2.a5').version, [2, 'a', 5]) - def test_unique_ordered_append(self): + def test_unique_ordered_extend(self): """Test unique_ordered_list_append method""" base = ["potato", "tomato", "orange"] reference = ["potato", "tomato", "orange", "apple"] - self.assertEqual(tu.unique_ordered_append(base, ["apple"]), reference) - self.assertEqual(tu.unique_ordered_append(base, ["apple", "apple"]), reference) - self.assertEqual(tu.unique_ordered_append(base, "apple"), reference) - self.assertNotEqual(tu.unique_ordered_append(base, "apple"), sorted(reference)) + self.assertEqual(tu.unique_ordered_extend(base, ["apple"]), reference) + self.assertEqual(tu.unique_ordered_extend(base, ["apple", "apple"]), reference) + self.assertEqual(tu.unique_ordered_extend(base, "apple"), reference) + self.assertNotEqual(tu.unique_ordered_extend(base, "apple"), sorted(reference)) error_pattern = "given extra list is not iterable" - self.assertErrorRegex(EasyBuildError, error_pattern, tu.unique_ordered_append, base, 0) + self.assertErrorRegex(EasyBuildError, error_pattern, tu.unique_ordered_extend, base, 0) base = "potato" error_pattern = "given base cannot be extended" - self.assertErrorRegex(EasyBuildError, error_pattern, tu.unique_ordered_append, base, reference) + self.assertErrorRegex(EasyBuildError, error_pattern, tu.unique_ordered_extend, base, reference) def suite(): From 485843df7fadf1773e8e1c5d6a80bb424f21cff8 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 2 Oct 2024 23:08:58 +0200 Subject: [PATCH 33/33] create local copy of base list in unique_ordered_extend --- easybuild/tools/utilities.py | 17 +++++++++-------- test/framework/utilities_test.py | 11 +++++++---- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index 0b7b50475f..ddfeb2074b 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -223,20 +223,21 @@ def nub(list_): return [x for x in list_ if x not in seen and not seen_add(x)] -def unique_ordered_extend(base, extra): - """Extend base list with elements of extra list keeping order and without duplicates""" - if isinstance(extra, str): - # avoid strings as they are iterables and generate wrong result without error - raise EasyBuildError(f"given extra list is a string: {extra}") +def unique_ordered_extend(base, affix): + """Extend base list with elements of affix list keeping order and without duplicates""" + if isinstance(affix, str): + # avoid extending with strings, as iterables generate wrong result without error + raise EasyBuildError(f"given affix list is a string: {affix}") try: - base.extend(extra) + ext_base = base.copy() + ext_base.extend(affix) except TypeError as err: - raise EasyBuildError(f"given extra list is not iterable: {extra}") from err + raise EasyBuildError(f"given affix list is not iterable: {affix}") from err except AttributeError as err: raise EasyBuildError(f"given base cannot be extended: {base}") from err - return nub(base) # remove duplicates + return nub(ext_base) # remove duplicates def get_class_for(modulepath, class_name): diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py index f1df4bb85f..fdb3488501 100644 --- a/test/framework/utilities_test.py +++ b/test/framework/utilities_test.py @@ -194,16 +194,19 @@ def test_LooseVersion(self): def test_unique_ordered_extend(self): """Test unique_ordered_list_append method""" base = ["potato", "tomato", "orange"] + base_orig = base.copy() reference = ["potato", "tomato", "orange", "apple"] self.assertEqual(tu.unique_ordered_extend(base, ["apple"]), reference) self.assertEqual(tu.unique_ordered_extend(base, ["apple", "apple"]), reference) - self.assertEqual(tu.unique_ordered_extend(base, "apple"), reference) - self.assertNotEqual(tu.unique_ordered_extend(base, "apple"), sorted(reference)) + self.assertNotEqual(tu.unique_ordered_extend(base, ["apple"]), sorted(reference)) + # original list should not be modified + self.assertEqual(base, base_orig) - error_pattern = "given extra list is not iterable" + error_pattern = "given affix list is a string" + self.assertErrorRegex(EasyBuildError, error_pattern, tu.unique_ordered_extend, base, "apple") + error_pattern = "given affix list is not iterable" self.assertErrorRegex(EasyBuildError, error_pattern, tu.unique_ordered_extend, base, 0) - base = "potato" error_pattern = "given base cannot be extended" self.assertErrorRegex(EasyBuildError, error_pattern, tu.unique_ordered_extend, base, reference)