diff --git a/easybuild/toolchains/fcc.py b/easybuild/toolchains/fcc.py index 038b7540c3..0da1fc149f 100644 --- a/easybuild/toolchains/fcc.py +++ b/easybuild/toolchains/fcc.py @@ -40,39 +40,10 @@ class FCC(FujitsuCompiler): OPTIONAL = False # override in order to add an exception for the Fujitsu lang/tcsds module - def _add_dependency_variables(self, names=None, cpp=None, ld=None): - """ Add LDFLAGS and CPPFLAGS to the self.variables based on the dependencies - names should be a list of strings containing the name of the dependency + def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None): """ - cpp_paths = ['include'] - ld_paths = ['lib64', 'lib'] - - if cpp is not None: - for p in cpp: - if p not in cpp_paths: - cpp_paths.append(p) - if ld is not None: - for p in ld: - if p not in ld_paths: - ld_paths.append(p) - - if not names: - deps = self.dependencies - else: - deps = [{'name': name} for name in names if name is not None] - - # collect software install prefixes for dependencies - roots = [] - for dep in deps: - if dep.get('external_module', False): - # for software names provided via external modules, install prefix may be unknown - names = dep['external_module_metadata'].get('name', []) - roots.extend([root for root in self.get_software_root(names) if root is not None]) - else: - roots.extend(self.get_software_root(dep['name'])) - - for root in roots: - # skip Fujitsu's 'lang/tcsds' module, including the top level include breaks vectorization in clang mode - if 'tcsds' not in root: - self.variables.append_subdirs("CPPFLAGS", root, subdirs=cpp_paths) - self.variables.append_subdirs("LDFLAGS", root, subdirs=ld_paths) + Append prepocessor paths for given dependency root directory + """ + # skip Fujitsu's 'lang/tcsds' module, including the top level include breaks vectorization in clang mode + if "tcsds" not in dep_root: + super()._add_dependency_cpp_headers(dep_root, extra_dirs=extra_dirs) diff --git a/easybuild/toolchains/linalg/acml.py b/easybuild/toolchains/linalg/acml.py index 1470b1f360..e38424726b 100644 --- a/easybuild/toolchains/linalg/acml.py +++ b/easybuild/toolchains/linalg/acml.py @@ -77,9 +77,9 @@ def _set_blas_variables(self): for root in self.get_software_root(self.BLAS_MODULE_NAME): subdirs = self.ACML_SUBDIRS_MAP[self.COMPILER_FAMILY] self.BLAS_LIB_DIR = [os.path.join(x, 'lib') for x in subdirs] - self.variables.append_exists('LDFLAGS', root, self.BLAS_LIB_DIR, append_all=True) + self._add_dependency_linker_paths(root, extra_dirs=self.BLAS_LIB_DIR) incdirs = [os.path.join(x, 'include') for x in subdirs] - self.variables.append_exists('CPPFLAGS', root, incdirs, append_all=True) + self._add_dependency_cpp_headers(root, extra_dirs=incdirs) except Exception: raise EasyBuildError("_set_blas_variables: ACML set LDFLAGS/CPPFLAGS unknown entry in ACML_SUBDIRS_MAP " "with compiler family %s", self.COMPILER_FAMILY) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 08c6cbf776..3e06077d35 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -265,6 +265,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'rpath_filter', 'rpath_override_dirs', 'required_linked_shared_libs', + 'search_path_cpp_headers', 'skip', 'software_commit', 'stop', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index fd5a5844c1..1efa53e0bd 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -102,6 +102,7 @@ from easybuild.tools.run import run_shell_cmd from easybuild.tools.package.utilities import avail_package_naming_schemes from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, OPTARCH_MAP_CHAR, OPTARCH_SEP, Compiler +from easybuild.tools.toolchain.toolchain import SEARCH_PATH_CPP_HEADERS, DEFAULT_SEARCH_PATH_CPP_HEADERS from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME from easybuild.tools.repository.repository import avail_repositories from easybuild.tools.systemtools import DARWIN, UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family @@ -637,6 +638,8 @@ def config_options(self): "(is passed as list of arguments to create the repository instance). " "For more info, use --avail-repositories."), 'strlist', 'store', self.default_repositorypath), + 'search-path-cpp-headers': ("Search path used at build time for include directories", 'choice', + 'store', DEFAULT_SEARCH_PATH_CPP_HEADERS, [*SEARCH_PATH_CPP_HEADERS]), 'sourcepath': ("Path(s) to where sources should be downloaded (string, colon-separated)", None, 'store', mk_full_default_path('sourcepath')), 'subdir-modules': ("Installpath subdir for modules", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_modules']), diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index de69214388..7f768a134a 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -93,6 +93,7 @@ class Compiler(Toolchain): 'vectorize': (None, "Enable compiler auto-vectorization, default except for noopt and lowopt"), 'packed-linker-options': (False, "Pack the linker options as comma separated list"), # ScaLAPACK mainly 'rpath': (True, "Use RPATH wrappers when --rpath is enabled in EasyBuild configuration"), + 'search-path-cpp-headers': (None, "Search path used at build time for include directories"), 'extra_cflags': (None, "Specify extra CFLAGS options."), 'extra_cxxflags': (None, "Specify extra CXXFLAGS options."), 'extra_fflags': (None, "Specify extra FFLAGS options."), diff --git a/easybuild/tools/toolchain/constants.py b/easybuild/tools/toolchain/constants.py index 643e8b4d21..72b333cf43 100644 --- a/easybuild/tools/toolchain/constants.py +++ b/easybuild/tools/toolchain/constants.py @@ -31,9 +31,9 @@ * Kenneth Hoste (Ghent University) """ +from easybuild.tools.toolchain.variables import CommandFlagList, CommaSharedLibs, CommaStaticLibs, FlagList +from easybuild.tools.toolchain.variables import IncludePaths, LibraryList, LinkLibraryPaths, SearchPaths from easybuild.tools.variables import AbsPathList -from easybuild.tools.toolchain.variables import CommandFlagList, CommaSharedLibs, CommaStaticLibs -from easybuild.tools.toolchain.variables import FlagList, IncludePaths, LibraryList, LinkLibraryPaths COMPILER_VARIABLES = [ @@ -65,7 +65,13 @@ ('LDFLAGS', 'Flags passed to linker'), # TODO: overridden by command line? ], IncludePaths: [ - ('CPPFLAGS', 'Precompiler flags'), + ('CPPFLAGS', 'Preprocessor flags'), + ], + SearchPaths: [ + ('CPATH', 'Location of C/C++ header files'), + ('C_INCLUDE_PATH', 'Location of C header files'), + ('CPLUS_INCLUDE_PATH', 'Location of C++ header files'), + ('OBJC_INCLUDE_PATH', 'Location of Objective C header files'), ], CommandFlagList: COMPILER_VARIABLES, } diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 3dd76903b6..bf1ddd8f1e 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -69,7 +69,7 @@ from easybuild.tools.systemtools import LINUX, get_os_type from easybuild.tools.toolchain.options import ToolchainOptions from easybuild.tools.toolchain.toolchainvariables import ToolchainVariables -from easybuild.tools.utilities import nub, trace_msg +from easybuild.tools.utilities import nub, unique_ordered_extend, trace_msg _log = fancylogger.getLogger('tools.toolchain', fname=False) @@ -95,6 +95,17 @@ TOOLCHAIN_CAPABILITY_LAPACK_FAMILY, TOOLCHAIN_CAPABILITY_MPI_FAMILY, ] +# modes to handle CPP header search paths +# see: https://gcc.gnu.org/onlinedocs/cpp/Environment-Variables.html +SEARCH_PATH_CPP_HEADERS_FLAGS = "CPPFLAGS" +SEARCH_PATH_CPP_HEADERS_CPATH = "CPATH" +SEARCH_PATH_CPP_HEADERS_INCLUDE = "INCLUDE_PATHS" +SEARCH_PATH_CPP_HEADERS = { + SEARCH_PATH_CPP_HEADERS_FLAGS: ["CPPFLAGS"], + SEARCH_PATH_CPP_HEADERS_CPATH: ["CPATH"], + SEARCH_PATH_CPP_HEADERS_INCLUDE: ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"], +} +DEFAULT_SEARCH_PATH_CPP_HEADERS = SEARCH_PATH_CPP_HEADERS_FLAGS def is_system_toolchain(tc_name): @@ -850,7 +861,7 @@ def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True, else: self.log.debug("prepare: set additional variables onlymod=%s", onlymod) - # add LDFLAGS and CPPFLAGS from dependencies to self.vars + # add linker and preprocessor paths of dependencies to self.vars self._add_dependency_variables() self.generate_vars() self._setenv_variables(onlymod, verbose=not silent) @@ -1049,39 +1060,71 @@ def handle_sysroot(self): setvar('PKG_CONFIG_PATH', os.pathsep.join(pkg_config_path)) def _add_dependency_variables(self, names=None, cpp=None, ld=None): - """ Add LDFLAGS and CPPFLAGS to the self.variables based on the dependencies - names should be a list of strings containing the name of the dependency """ - cpp_paths = ['include'] - ld_paths = ['lib64', 'lib'] - - if cpp is not None: - for p in cpp: - if p not in cpp_paths: - cpp_paths.append(p) - if ld is not None: - for p in ld: - if p not in ld_paths: - ld_paths.append(p) - - if not names: - deps = self.dependencies - else: - deps = [{'name': name} for name in names if name is not None] + Add linker and preprocessor paths of dependencies to self.variables + :names: list of strings containing the name of the dependency + """ + # collect dependencies + dependencies = self.dependencies if names is None else [{"name": name} for name in names if name] # collect software install prefixes for dependencies - roots = [] - for dep in deps: - if dep.get('external_module', False): + dependency_roots = [] + for dep in dependencies: + if dep.get("external_module", False): # for software names provided via external modules, install prefix may be unknown - names = dep['external_module_metadata'].get('name', []) - roots.extend([root for root in self.get_software_root(names) if root is not None]) + names = dep["external_module_metadata"].get("name", []) + dependency_roots.extend([root for root in self.get_software_root(names) if root is not None]) else: - roots.extend(self.get_software_root(dep['name'])) + dependency_roots.extend(self.get_software_root(dep["name"])) + + for root in dependency_roots: + self._add_dependency_cpp_headers(root, extra_dirs=cpp) + self._add_dependency_linker_paths(root, extra_dirs=ld) + + def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None): + """ + Append prepocessor paths for given dependency root directory + """ + if extra_dirs is None: + extra_dirs = () + + header_dirs = ["include"] + header_dirs = unique_ordered_extend(header_dirs, extra_dirs) + + # mode of operation is defined by search-path-cpp-headers option + # toolchain option has precedence over build option + cpp_headers_mode = DEFAULT_SEARCH_PATH_CPP_HEADERS + build_opt = build_option("search_path_cpp_headers") + if self.options.get("search-path-cpp-headers") is not None: + cpp_headers_mode = self.options.option("search-path-cpp-headers") + self.log.debug("search-path-cpp-headers set by toolchain option: %s", cpp_headers_mode) + elif build_opt is not None: + cpp_headers_mode = build_opt + self.log.debug("search-path-cpp-headers set by build option: %s", cpp_headers_mode) + + if cpp_headers_mode not in SEARCH_PATH_CPP_HEADERS: + raise EasyBuildError( + "Unknown value selected for option search-path-cpp-headers: %s. Choose one of: %s", + cpp_headers_mode, ", ".join(SEARCH_PATH_CPP_HEADERS) + ) + + for env_var in SEARCH_PATH_CPP_HEADERS[cpp_headers_mode]: + self.log.debug("Adding header paths to toolchain variable '%s': %s", env_var, dep_root) + self.variables.append_subdirs(env_var, dep_root, subdirs=header_dirs) + + def _add_dependency_linker_paths(self, dep_root, extra_dirs=None): + """ + Append linker paths for given dependency root directory + """ + if extra_dirs is None: + extra_dirs = () + + lib_dirs = ["lib64", "lib"] + lib_dirs = unique_ordered_extend(lib_dirs, extra_dirs) - for root in roots: - self.variables.append_subdirs("CPPFLAGS", root, subdirs=cpp_paths) - self.variables.append_subdirs("LDFLAGS", root, subdirs=ld_paths) + env_var = "LDFLAGS" + self.log.debug("Adding lib paths to toolchain variable '%s': %s", env_var, dep_root) + self.variables.append_subdirs(env_var, dep_root, subdirs=lib_dirs) def _setenv_variables(self, donotset=None, verbose=True): """Actually set the environment variables""" diff --git a/easybuild/tools/toolchain/variables.py b/easybuild/tools/toolchain/variables.py index 482f989480..5e051d4331 100644 --- a/easybuild/tools/toolchain/variables.py +++ b/easybuild/tools/toolchain/variables.py @@ -46,6 +46,11 @@ class LinkLibraryPaths(AbsPathList): PREFIX = '-L' +class SearchPaths(AbsPathList): + """Colon-separated list of absolute paths""" + SEPARATOR = ':' + + class FlagList(StrList): """Flag list""" PREFIX = "-" diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index fe254ea4d2..ddfeb2074b 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -223,6 +223,23 @@ def nub(list_): return [x for x in list_ if x not in seen and not seen_add(x)] +def unique_ordered_extend(base, affix): + """Extend base list with elements of affix list keeping order and without duplicates""" + if isinstance(affix, str): + # avoid extending with strings, as iterables generate wrong result without error + raise EasyBuildError(f"given affix list is a string: {affix}") + + try: + ext_base = base.copy() + ext_base.extend(affix) + except TypeError as err: + raise EasyBuildError(f"given affix list is not iterable: {affix}") from err + except AttributeError as err: + raise EasyBuildError(f"given base cannot be extended: {base}") from err + + return nub(ext_base) # remove duplicates + + def get_class_for(modulepath, class_name): """ Get class for a given Python class name and Python module path. diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 2e8c7893b1..779b0ed182 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -955,6 +955,49 @@ def test_precision_flags(self): self.modtool.purge() + def test_search_path_cpp_headers(self): + """Test functionality behind search-path-cpp-headers option""" + cpp_headers_mode = { + "CPPFLAGS": ["CPPFLAGS"], + "CPATH": ["CPATH"], + "INCLUDE_PATHS": ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"], + } + # test without toolchain option + for build_opt in cpp_headers_mode: + init_config(build_options={"search_path_cpp_headers": build_opt, "silent": True}) + tc = self.get_toolchain("foss", version="2018a") + with self.mocked_stdout_stderr(): + tc.prepare() + for env_var in cpp_headers_mode[build_opt]: + assert_fail_msg = ( + f"Variable {env_var} required by search-path-cpp-headers build option '{build_opt}' " + "not found in toolchain environment" + ) + self.assertIn(env_var, tc.variables, assert_fail_msg) + self.modtool.purge() + # test with toolchain option + for build_opt in cpp_headers_mode: + init_config(build_options={"search_path_cpp_headers": build_opt, "silent": True}) + for tc_opt in cpp_headers_mode: + tc = self.get_toolchain("foss", version="2018a") + tc.set_options({"search-path-cpp-headers": tc_opt}) + with self.mocked_stdout_stderr(): + tc.prepare() + for env_var in cpp_headers_mode[tc_opt]: + assert_fail_msg = ( + f"Variable {env_var} required by search-path-cpp-headers toolchain option '{tc_opt}' " + "not found in toolchain environment" + ) + self.assertIn(env_var, tc.variables, assert_fail_msg) + self.modtool.purge() + # test wrong toolchain option + tc = self.get_toolchain("foss", version="2018a") + tc.set_options({"search-path-cpp-headers": "WRONG_MODE"}) + with self.mocked_stdout_stderr(): + error_pattern = "Unknown value selected for option search-path-cpp-headers" + self.assertErrorRegex(EasyBuildError, error_pattern, tc.prepare) + self.modtool.purge() + def test_cgoolf_toolchain(self): """Test for cgoolf toolchain.""" tc = self.get_toolchain("cgoolf", version="1.1.6") diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py index 7a10692532..535b78c242 100644 --- a/test/framework/utilities_test.py +++ b/test/framework/utilities_test.py @@ -36,10 +36,10 @@ from datetime import datetime from unittest import TextTestRunner +import easybuild.tools.utilities as tu from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.utilities import time2str, natural_keys class UtilitiesTest(EnhancedTestCase): @@ -77,10 +77,10 @@ def test_time2str(self): (datetime(2019, 8, 5, 20, 39, 44), "159 hours 25 mins 21 secs"), ] for end, expected in test_cases: - self.assertEqual(time2str(end - start), expected) + self.assertEqual(tu.time2str(end - start), expected) error_pattern = "Incorrect value type provided to time2str, should be datetime.timedelta: <.* 'int'>" - self.assertErrorRegex(EasyBuildError, error_pattern, time2str, 123) + self.assertErrorRegex(EasyBuildError, error_pattern, tu.time2str, 123) def test_natural_keys(self): """Test the natural_keys function""" @@ -98,7 +98,7 @@ def test_natural_keys(self): ] shuffled_items = sorted_items[:] random.shuffle(shuffled_items) - shuffled_items.sort(key=natural_keys) + shuffled_items.sort(key=tu.natural_keys) self.assertEqual(shuffled_items, sorted_items) def test_LooseVersion(self): @@ -203,6 +203,26 @@ def test_LooseVersion(self): self.assertEqual(LooseVersion('2.a').version, [2, 'a']) self.assertEqual(LooseVersion('2.a5').version, [2, 'a', 5]) + def test_unique_ordered_extend(self): + """Test unique_ordered_list_append method""" + base = ["potato", "tomato", "orange"] + base_orig = base.copy() + + reference = ["potato", "tomato", "orange", "apple"] + self.assertEqual(tu.unique_ordered_extend(base, ["apple"]), reference) + self.assertEqual(tu.unique_ordered_extend(base, ["apple", "apple"]), reference) + self.assertNotEqual(tu.unique_ordered_extend(base, ["apple"]), sorted(reference)) + # original list should not be modified + self.assertEqual(base, base_orig) + + error_pattern = "given affix list is a string" + self.assertErrorRegex(EasyBuildError, error_pattern, tu.unique_ordered_extend, base, "apple") + error_pattern = "given affix list is not iterable" + self.assertErrorRegex(EasyBuildError, error_pattern, tu.unique_ordered_extend, base, 0) + base = "potato" + error_pattern = "given base cannot be extended" + self.assertErrorRegex(EasyBuildError, error_pattern, tu.unique_ordered_extend, base, reference) + def suite(): """ return all the tests in this file """