Merge pull request #4645 from lexming/cpath

add `--search-path-cpp-headers` configuration option to control how EasyBuild sets paths to headers at build time

easybuild/toolchains/fcc.py

@@ -40,39 +40,10 @@ class FCC(FujitsuCompiler):
     OPTIONAL = False
 
     # override in order to add an exception for the Fujitsu lang/tcsds module
-    def _add_dependency_variables(self, names=None, cpp=None, ld=None):
-        """ Add LDFLAGS and CPPFLAGS to the self.variables based on the dependencies
-            names should be a list of strings containing the name of the dependency
+    def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None):
         """
-        cpp_paths = ['include']
-        ld_paths = ['lib64', 'lib']
-
-        if cpp is not None:
-            for p in cpp:
-                if p not in cpp_paths:
-                    cpp_paths.append(p)
-        if ld is not None:
-            for p in ld:
-                if p not in ld_paths:
-                    ld_paths.append(p)
-
-        if not names:
-            deps = self.dependencies
-        else:
-            deps = [{'name': name} for name in names if name is not None]
-
-        # collect software install prefixes for dependencies
-        roots = []
-        for dep in deps:
-            if dep.get('external_module', False):
-                # for software names provided via external modules, install prefix may be unknown
-                names = dep['external_module_metadata'].get('name', [])
-                roots.extend([root for root in self.get_software_root(names) if root is not None])
-            else:
-                roots.extend(self.get_software_root(dep['name']))
-
-        for root in roots:
-            # skip Fujitsu's 'lang/tcsds' module, including the top level include breaks vectorization in clang mode
-            if 'tcsds' not in root:
-                self.variables.append_subdirs("CPPFLAGS", root, subdirs=cpp_paths)
-            self.variables.append_subdirs("LDFLAGS", root, subdirs=ld_paths)
+        Append prepocessor paths for given dependency root directory
+        """
+        # skip Fujitsu's 'lang/tcsds' module, including the top level include breaks vectorization in clang mode
+        if "tcsds" not in dep_root:
+            super()._add_dependency_cpp_headers(dep_root, extra_dirs=extra_dirs)

easybuild/toolchains/linalg/acml.py

@@ -77,9 +77,9 @@ def _set_blas_variables(self):
             for root in self.get_software_root(self.BLAS_MODULE_NAME):
                 subdirs = self.ACML_SUBDIRS_MAP[self.COMPILER_FAMILY]
                 self.BLAS_LIB_DIR = [os.path.join(x, 'lib') for x in subdirs]
-                self.variables.append_exists('LDFLAGS', root, self.BLAS_LIB_DIR, append_all=True)
+                self._add_dependency_linker_paths(root, extra_dirs=self.BLAS_LIB_DIR)
                 incdirs = [os.path.join(x, 'include') for x in subdirs]
-                self.variables.append_exists('CPPFLAGS', root, incdirs, append_all=True)
+                self._add_dependency_cpp_headers(root, extra_dirs=incdirs)
         except Exception:
             raise EasyBuildError("_set_blas_variables: ACML set LDFLAGS/CPPFLAGS unknown entry in ACML_SUBDIRS_MAP "
                                  "with compiler family %s", self.COMPILER_FAMILY)

easybuild/tools/config.py

@@ -265,6 +265,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'rpath_filter',
         'rpath_override_dirs',
         'required_linked_shared_libs',
+        'search_path_cpp_headers',
         'skip',
         'software_commit',
         'stop',

easybuild/tools/options.py

@@ -102,6 +102,7 @@
 from easybuild.tools.run import run_shell_cmd
 from easybuild.tools.package.utilities import avail_package_naming_schemes
 from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, OPTARCH_MAP_CHAR, OPTARCH_SEP, Compiler
+from easybuild.tools.toolchain.toolchain import SEARCH_PATH_CPP_HEADERS, DEFAULT_SEARCH_PATH_CPP_HEADERS
 from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME
 from easybuild.tools.repository.repository import avail_repositories
 from easybuild.tools.systemtools import DARWIN, UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family
@@ -637,6 +638,8 @@ def config_options(self):
                                 "(is passed as list of arguments to create the repository instance). "
                                 "For more info, use --avail-repositories."),
                                'strlist', 'store', self.default_repositorypath),
+            'search-path-cpp-headers': ("Search path used at build time for include directories", 'choice',
+                                        'store', DEFAULT_SEARCH_PATH_CPP_HEADERS, [*SEARCH_PATH_CPP_HEADERS]),
             'sourcepath': ("Path(s) to where sources should be downloaded (string, colon-separated)",
                            None, 'store', mk_full_default_path('sourcepath')),
             'subdir-modules': ("Installpath subdir for modules", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_modules']),

easybuild/tools/toolchain/compiler.py

@@ -93,6 +93,7 @@ class Compiler(Toolchain):
         'vectorize': (None, "Enable compiler auto-vectorization, default except for noopt and lowopt"),
         'packed-linker-options': (False, "Pack the linker options as comma separated list"),  # ScaLAPACK mainly
         'rpath': (True, "Use RPATH wrappers when --rpath is enabled in EasyBuild configuration"),
+        'search-path-cpp-headers': (None, "Search path used at build time for include directories"),
         'extra_cflags': (None, "Specify extra CFLAGS options."),
         'extra_cxxflags': (None, "Specify extra CXXFLAGS options."),
         'extra_fflags': (None, "Specify extra FFLAGS options."),

easybuild/tools/toolchain/constants.py

@@ -31,9 +31,9 @@
 * Kenneth Hoste (Ghent University)
 """
 
+from easybuild.tools.toolchain.variables import CommandFlagList, CommaSharedLibs, CommaStaticLibs, FlagList
+from easybuild.tools.toolchain.variables import IncludePaths, LibraryList, LinkLibraryPaths, SearchPaths
 from easybuild.tools.variables import AbsPathList
-from easybuild.tools.toolchain.variables import CommandFlagList, CommaSharedLibs, CommaStaticLibs
-from easybuild.tools.toolchain.variables import FlagList, IncludePaths, LibraryList, LinkLibraryPaths
 
 
 COMPILER_VARIABLES = [
@@ -65,7 +65,13 @@
         ('LDFLAGS', 'Flags passed to linker'),  # TODO: overridden by command line?
     ],
     IncludePaths: [
-        ('CPPFLAGS', 'Precompiler flags'),
+        ('CPPFLAGS', 'Preprocessor flags'),
+    ],
+    SearchPaths: [
+        ('CPATH', 'Location of C/C++ header files'),
+        ('C_INCLUDE_PATH', 'Location of C header files'),
+        ('CPLUS_INCLUDE_PATH', 'Location of C++ header files'),
+        ('OBJC_INCLUDE_PATH', 'Location of Objective C header files'),
     ],
     CommandFlagList: COMPILER_VARIABLES,
 }

easybuild/tools/toolchain/toolchain.py

@@ -69,7 +69,7 @@
 from easybuild.tools.systemtools import LINUX, get_os_type
 from easybuild.tools.toolchain.options import ToolchainOptions
 from easybuild.tools.toolchain.toolchainvariables import ToolchainVariables
-from easybuild.tools.utilities import nub, trace_msg
+from easybuild.tools.utilities import nub, unique_ordered_extend, trace_msg
 
 
 _log = fancylogger.getLogger('tools.toolchain', fname=False)
@@ -95,6 +95,17 @@
     TOOLCHAIN_CAPABILITY_LAPACK_FAMILY,
     TOOLCHAIN_CAPABILITY_MPI_FAMILY,
 ]
+# modes to handle CPP header search paths
+# see: https://gcc.gnu.org/onlinedocs/cpp/Environment-Variables.html
+SEARCH_PATH_CPP_HEADERS_FLAGS = "CPPFLAGS"
+SEARCH_PATH_CPP_HEADERS_CPATH = "CPATH"
+SEARCH_PATH_CPP_HEADERS_INCLUDE = "INCLUDE_PATHS"
+SEARCH_PATH_CPP_HEADERS = {
+    SEARCH_PATH_CPP_HEADERS_FLAGS: ["CPPFLAGS"],
+    SEARCH_PATH_CPP_HEADERS_CPATH: ["CPATH"],
+    SEARCH_PATH_CPP_HEADERS_INCLUDE: ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"],
+}
+DEFAULT_SEARCH_PATH_CPP_HEADERS = SEARCH_PATH_CPP_HEADERS_FLAGS
 
 
 def is_system_toolchain(tc_name):
@@ -850,7 +861,7 @@ def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True,
             else:
                 self.log.debug("prepare: set additional variables onlymod=%s", onlymod)
 
-                # add LDFLAGS and CPPFLAGS from dependencies to self.vars
+                # add linker and preprocessor paths of dependencies to self.vars
                 self._add_dependency_variables()
                 self.generate_vars()
                 self._setenv_variables(onlymod, verbose=not silent)
@@ -1049,39 +1060,71 @@ def handle_sysroot(self):
                 setvar('PKG_CONFIG_PATH', os.pathsep.join(pkg_config_path))
 
     def _add_dependency_variables(self, names=None, cpp=None, ld=None):
-        """ Add LDFLAGS and CPPFLAGS to the self.variables based on the dependencies
-            names should be a list of strings containing the name of the dependency
         """
-        cpp_paths = ['include']
-        ld_paths = ['lib64', 'lib']
-
-        if cpp is not None:
-            for p in cpp:
-                if p not in cpp_paths:
-                    cpp_paths.append(p)
-        if ld is not None:
-            for p in ld:
-                if p not in ld_paths:
-                    ld_paths.append(p)
-
-        if not names:
-            deps = self.dependencies
-        else:
-            deps = [{'name': name} for name in names if name is not None]
+        Add linker and preprocessor paths of dependencies to self.variables
+        :names: list of strings containing the name of the dependency
+        """
+        # collect dependencies
+        dependencies = self.dependencies if names is None else [{"name": name} for name in names if name]
 
         # collect software install prefixes for dependencies
-        roots = []
-        for dep in deps:
-            if dep.get('external_module', False):
+        dependency_roots = []
+        for dep in dependencies:
+            if dep.get("external_module", False):
                 # for software names provided via external modules, install prefix may be unknown
-                names = dep['external_module_metadata'].get('name', [])
-                roots.extend([root for root in self.get_software_root(names) if root is not None])
+                names = dep["external_module_metadata"].get("name", [])
+                dependency_roots.extend([root for root in self.get_software_root(names) if root is not None])
             else:
-                roots.extend(self.get_software_root(dep['name']))
+                dependency_roots.extend(self.get_software_root(dep["name"]))
+
+        for root in dependency_roots:
+            self._add_dependency_cpp_headers(root, extra_dirs=cpp)
+            self._add_dependency_linker_paths(root, extra_dirs=ld)
+
+    def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None):
+        """
+        Append prepocessor paths for given dependency root directory
+        """
+        if extra_dirs is None:
+            extra_dirs = ()
+
+        header_dirs = ["include"]
+        header_dirs = unique_ordered_extend(header_dirs, extra_dirs)
+
+        # mode of operation is defined by search-path-cpp-headers option
+        # toolchain option has precedence over build option
+        cpp_headers_mode = DEFAULT_SEARCH_PATH_CPP_HEADERS
+        build_opt = build_option("search_path_cpp_headers")
+        if self.options.get("search-path-cpp-headers") is not None:
+            cpp_headers_mode = self.options.option("search-path-cpp-headers")
+            self.log.debug("search-path-cpp-headers set by toolchain option: %s", cpp_headers_mode)
+        elif build_opt is not None:
+            cpp_headers_mode = build_opt
+            self.log.debug("search-path-cpp-headers set by build option: %s", cpp_headers_mode)
+
+        if cpp_headers_mode not in SEARCH_PATH_CPP_HEADERS:
+            raise EasyBuildError(
+                "Unknown value selected for option search-path-cpp-headers: %s. Choose one of: %s",
+                cpp_headers_mode, ", ".join(SEARCH_PATH_CPP_HEADERS)
+            )
+
+        for env_var in SEARCH_PATH_CPP_HEADERS[cpp_headers_mode]:
+            self.log.debug("Adding header paths to toolchain variable '%s': %s", env_var, dep_root)
+            self.variables.append_subdirs(env_var, dep_root, subdirs=header_dirs)
+
+    def _add_dependency_linker_paths(self, dep_root, extra_dirs=None):
+        """
+        Append linker paths for given dependency root directory
+        """
+        if extra_dirs is None:
+            extra_dirs = ()
+
+        lib_dirs = ["lib64", "lib"]
+        lib_dirs = unique_ordered_extend(lib_dirs, extra_dirs)
 
-        for root in roots:
-            self.variables.append_subdirs("CPPFLAGS", root, subdirs=cpp_paths)
-            self.variables.append_subdirs("LDFLAGS", root, subdirs=ld_paths)
+        env_var = "LDFLAGS"
+        self.log.debug("Adding lib paths to toolchain variable '%s': %s", env_var, dep_root)
+        self.variables.append_subdirs(env_var, dep_root, subdirs=lib_dirs)
 
     def _setenv_variables(self, donotset=None, verbose=True):
         """Actually set the environment variables"""

easybuild/tools/toolchain/variables.py

@@ -46,6 +46,11 @@ class LinkLibraryPaths(AbsPathList):
     PREFIX = '-L'
 
 
+class SearchPaths(AbsPathList):
+    """Colon-separated list of absolute paths"""
+    SEPARATOR = ':'
+
+
 class FlagList(StrList):
     """Flag list"""
     PREFIX = "-"

easybuild/tools/utilities.py

@@ -223,6 +223,23 @@ def nub(list_):
     return [x for x in list_ if x not in seen and not seen_add(x)]
 
 
+def unique_ordered_extend(base, affix):
+    """Extend base list with elements of affix list keeping order and without duplicates"""
+    if isinstance(affix, str):
+        # avoid extending with strings, as iterables generate wrong result without error
+        raise EasyBuildError(f"given affix list is a string: {affix}")
+
+    try:
+        ext_base = base.copy()
+        ext_base.extend(affix)
+    except TypeError as err:
+        raise EasyBuildError(f"given affix list is not iterable: {affix}") from err
+    except AttributeError as err:
+        raise EasyBuildError(f"given base cannot be extended: {base}") from err
+
+    return nub(ext_base)  # remove duplicates
+
+
 def get_class_for(modulepath, class_name):
     """
     Get class for a given Python class name and Python module path.

test/framework/toolchain.py

@@ -955,6 +955,49 @@ def test_precision_flags(self):
 
                 self.modtool.purge()
 
+    def test_search_path_cpp_headers(self):
+        """Test functionality behind search-path-cpp-headers option"""
+        cpp_headers_mode = {
+            "CPPFLAGS": ["CPPFLAGS"],
+            "CPATH": ["CPATH"],
+            "INCLUDE_PATHS": ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"],
+        }
+        # test without toolchain option
+        for build_opt in cpp_headers_mode:
+            init_config(build_options={"search_path_cpp_headers": build_opt, "silent": True})
+            tc = self.get_toolchain("foss", version="2018a")
+            with self.mocked_stdout_stderr():
+                tc.prepare()
+                for env_var in cpp_headers_mode[build_opt]:
+                    assert_fail_msg = (
+                        f"Variable {env_var} required by search-path-cpp-headers build option '{build_opt}' "
+                        "not found in toolchain environment"
+                    )
+                    self.assertIn(env_var, tc.variables, assert_fail_msg)
+                self.modtool.purge()
+        # test with toolchain option
+        for build_opt in cpp_headers_mode:
+            init_config(build_options={"search_path_cpp_headers": build_opt, "silent": True})
+            for tc_opt in cpp_headers_mode:
+                tc = self.get_toolchain("foss", version="2018a")
+                tc.set_options({"search-path-cpp-headers": tc_opt})
+                with self.mocked_stdout_stderr():
+                    tc.prepare()
+                    for env_var in cpp_headers_mode[tc_opt]:
+                        assert_fail_msg = (
+                            f"Variable {env_var} required by search-path-cpp-headers toolchain option '{tc_opt}' "
+                            "not found in toolchain environment"
+                        )
+                        self.assertIn(env_var, tc.variables, assert_fail_msg)
+                self.modtool.purge()
+        # test wrong toolchain option
+        tc = self.get_toolchain("foss", version="2018a")
+        tc.set_options({"search-path-cpp-headers": "WRONG_MODE"})
+        with self.mocked_stdout_stderr():
+            error_pattern = "Unknown value selected for option search-path-cpp-headers"
+            self.assertErrorRegex(EasyBuildError, error_pattern, tc.prepare)
+        self.modtool.purge()
+
     def test_cgoolf_toolchain(self):
         """Test for cgoolf toolchain."""
         tc = self.get_toolchain("cgoolf", version="1.1.6")

test/framework/utilities_test.py

@@ -36,10 +36,10 @@
 from datetime import datetime
 from unittest import TextTestRunner
 
+import easybuild.tools.utilities as tu
 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered
 from easybuild.tools import LooseVersion
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.utilities import time2str, natural_keys
 
 
 class UtilitiesTest(EnhancedTestCase):
@@ -77,10 +77,10 @@ def test_time2str(self):
             (datetime(2019, 8, 5, 20, 39, 44), "159 hours 25 mins 21 secs"),
         ]
         for end, expected in test_cases:
-            self.assertEqual(time2str(end - start), expected)
+            self.assertEqual(tu.time2str(end - start), expected)
 
         error_pattern = "Incorrect value type provided to time2str, should be datetime.timedelta: <.* 'int'>"
-        self.assertErrorRegex(EasyBuildError, error_pattern, time2str, 123)
+        self.assertErrorRegex(EasyBuildError, error_pattern, tu.time2str, 123)
 
     def test_natural_keys(self):
         """Test the natural_keys function"""
@@ -98,7 +98,7 @@ def test_natural_keys(self):
         ]
         shuffled_items = sorted_items[:]
         random.shuffle(shuffled_items)
-        shuffled_items.sort(key=natural_keys)
+        shuffled_items.sort(key=tu.natural_keys)
         self.assertEqual(shuffled_items, sorted_items)
 
     def test_LooseVersion(self):
@@ -203,6 +203,26 @@ def test_LooseVersion(self):
         self.assertEqual(LooseVersion('2.a').version, [2, 'a'])
         self.assertEqual(LooseVersion('2.a5').version, [2, 'a', 5])
 
+    def test_unique_ordered_extend(self):
+        """Test unique_ordered_list_append method"""
+        base = ["potato", "tomato", "orange"]
+        base_orig = base.copy()
+
+        reference = ["potato", "tomato", "orange", "apple"]
+        self.assertEqual(tu.unique_ordered_extend(base, ["apple"]), reference)
+        self.assertEqual(tu.unique_ordered_extend(base, ["apple", "apple"]), reference)
+        self.assertNotEqual(tu.unique_ordered_extend(base, ["apple"]), sorted(reference))
+        # original list should not be modified
+        self.assertEqual(base, base_orig)
+
+        error_pattern = "given affix list is a string"
+        self.assertErrorRegex(EasyBuildError, error_pattern, tu.unique_ordered_extend, base, "apple")
+        error_pattern = "given affix list is not iterable"
+        self.assertErrorRegex(EasyBuildError, error_pattern, tu.unique_ordered_extend, base, 0)
+        base = "potato"
+        error_pattern = "given base cannot be extended"
+        self.assertErrorRegex(EasyBuildError, error_pattern, tu.unique_ordered_extend, base, reference)
+
 
 def suite():
     """ return all the tests in this file """