Merge pull request #1472 from geimer/user_modules

Allow user-local modules with hierarchical naming schemes

easybuild/framework/easyblock.py

@@ -1036,12 +1036,21 @@ def make_module_extend_modpath(self):
         txt = ''
         if self.cfg['include_modpath_extensions']:
             modpath_exts = ActiveMNS().det_modpath_extensions(self.cfg)
-            self.log.debug("Including module path extensions returned by module naming scheme: %s" % modpath_exts)
+            self.log.debug("Including module path extensions returned by module naming scheme: %s", modpath_exts)
             full_path_modpath_extensions = [os.path.join(self.installdir_mod, ext) for ext in modpath_exts]
             # module path extensions must exist, otherwise loading this module file will fail
             for modpath_extension in full_path_modpath_extensions:
                 mkdir(modpath_extension, parents=True)
             txt = self.module_generator.use(full_path_modpath_extensions)
+
+            # add user-specific module path; use statement will be guarded so no need to create the directories
+            user_modpath = build_option('subdir_user_modules')
+            if user_modpath:
+                user_modpath_exts = ActiveMNS().det_user_modpath_extensions(self.cfg)
+                user_modpath_exts = [os.path.join(user_modpath, e) for e in user_modpath_exts]
+                self.log.debug("Including user module path extensions returned by naming scheme: %s", user_modpath_exts)
+                txt += self.module_generator.use(user_modpath_exts, prefix=self.module_generator.getenv_cmd('HOME'),
+                                                 guarded=True)
         else:
             self.log.debug("Not including module path extensions, as specified.")
         return txt

easybuild/framework/easyconfig/easyconfig.py

@@ -1283,6 +1283,13 @@ def det_modpath_extensions(self, ec):
         self.log.debug("Obtained modulepath extensions: %s" % modpath_extensions)
         return modpath_extensions
 
+    def det_user_modpath_extensions(self, ec):
+        """Determine user-specific modulepath extensions according to module naming scheme."""
+        self.log.debug("Determining user modulepath extensions for %s", ec)
+        modpath_extensions = self.mns.det_user_modpath_extensions(self.check_ec_type(ec))
+        self.log.debug("Obtained user modulepath extensions: %s", modpath_extensions)
+        return modpath_extensions
+
     def det_init_modulepaths(self, ec):
         """Determine initial modulepaths according to module naming scheme."""
         self.log.debug("Determining initial module paths for %s" % ec)

easybuild/tools/config.py

@@ -112,6 +112,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'regtest_output_dir',
         'skip',
         'stop',
+        'subdir_user_modules',
         'test_report_env_filter',
         'testoutput',
         'umask',

easybuild/tools/module_generator.py

@@ -128,6 +128,12 @@ def conditional_statement(self, condition, body, negative=False):
         """Return formatted conditional statement, with given condition and body."""
         raise NotImplementedError
 
+    def getenv_cmd(self, envvar):
+        """
+        Return module-syntax specific code to get value of specific environment variable.
+        """
+        raise NotImplementedError
+
     def load_module(self, mod_name, recursive_unload=False, unload_modules=None):
         """
         Generate load statement for specified module.
@@ -286,13 +292,26 @@ def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
         statements = [template % (key, p) for p in abspaths]
         return ''.join(statements)
 
-    def use(self, paths):
+    def use(self, paths, prefix=None, guarded=False):
         """
         Generate module use statements for given list of module paths.
+        @param paths: list of module path extensions to generate use statements for; paths will be quoted
+        @param prefix: optional path prefix; not quoted, i.e., can be a statement
+        @param guarded: use statements will be guarded to only apply if path exists
         """
         use_statements = []
         for path in paths:
-            use_statements.append("module use %s\n" % path)
+            quoted_path = quote_str(path)
+            if prefix:
+                full_path = '[ file join %s %s ]' % (prefix, quoted_path)
+            else:
+                full_path = quoted_path
+            if guarded:
+                cond_statement = self.conditional_statement('file isdirectory %s' % full_path,
+                                                            'module use %s' % full_path)
+                use_statements.append(cond_statement)
+            else:
+                use_statements.append("module use %s\n" % full_path)
         return ''.join(use_statements)
 
     def set_environment(self, key, value, relpath=False):
@@ -325,6 +344,12 @@ def set_alias(self, key, value):
         # quotes are needed, to ensure smooth working of EBDEVEL* modulefiles
         return 'set-alias\t%s\t\t%s\n' % (key, quote_str(value))
 
+    def getenv_cmd(self, envvar):
+        """
+        Return module-syntax specific code to get value of specific environment variable.
+        """
+        return '$env(%s)' % envvar
+
 
 class ModuleGeneratorLua(ModuleGenerator):
     """
@@ -339,7 +364,7 @@ class ModuleGeneratorLua(ModuleGenerator):
     LOAD_TEMPLATE = 'load("%(mod_name)s")'
 
     PATH_JOIN_TEMPLATE = 'pathJoin(root, "%s")'
-    PREPEND_PATH_TEMPLATE = 'prepend_path("%s", %s)\n'
+    PREPEND_PATH_TEMPLATE = 'prepend_path("%s", %s)'
 
     def __init__(self, *args, **kwargs):
         """ModuleGeneratorLua constructor."""
@@ -462,14 +487,30 @@ def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
                     abspaths.append('root')
 
         statements = [self.PREPEND_PATH_TEMPLATE % (key, p) for p in abspaths]
-        return ''.join(statements)
+        statements.append('')
+        return '\n'.join(statements)
 
-    def use(self, paths):
+    def use(self, paths, prefix=None, guarded=False):
         """
         Generate module use statements for given list of module paths.
-        @param paths: list of module path extensions to generate use statements for
+        @param paths: list of module path extensions to generate use statements for; paths will be quoted
+        @param prefix: optional path prefix; not quoted, i.e., can be a statement
+        @param guarded: use statements will be guarded to only apply if path exists
         """
-        return ''.join([self.PREPEND_PATH_TEMPLATE % ('MODULEPATH', quote_str(p)) for p in paths])
+        use_statements = []
+        for path in paths:
+            quoted_path = quote_str(path)
+            if prefix:
+                full_path = 'pathJoin(%s, %s)' % (prefix, quoted_path)
+            else:
+                full_path = quoted_path
+            if guarded:
+                cond_statement = self.conditional_statement('isDir(%s)' % full_path,
+                                                            self.PREPEND_PATH_TEMPLATE % ('MODULEPATH', full_path))
+                use_statements.append(cond_statement)
+            else:
+                use_statements.append(self.PREPEND_PATH_TEMPLATE % ('MODULEPATH', full_path) + '\n')
+        return ''.join(use_statements)
 
     def set_environment(self, key, value, relpath=False):
         """
@@ -497,6 +538,12 @@ def set_alias(self, key, value):
         # quotes are needed, to ensure smooth working of EBDEVEL* modulefiles
         return 'set_alias("%s", %s)\n' % (key, quote_str(value))
 
+    def getenv_cmd(self, envvar):
+        """
+        Return module-syntax specific code to get value of specific environment variable.
+        """
+        return 'os.getenv("%s")' % envvar
+
 
 def avail_module_generators():
     """

easybuild/tools/module_naming_scheme/categorized_hmns.py

@@ -83,6 +83,17 @@ def det_modpath_extensions(self, ec):
 
         return self.categorize_paths(basepaths)
 
+    def det_user_modpath_extensions(self, ec):
+        """
+        Determine user module path extensions, if any. As typical users are not expected to have many local modules,
+        further categorizing them using module classes is considered overkill. Thus, we are using a plain hierarchical
+        scheme for user modules instead.
+
+        Examples: Compiler/GCC/4.8.3 (for GCC/4.8.3 module), MPI/GCC/4.8.3/OpenMPI/1.6.5 (for OpenMPI/1.6.5 module)
+        """
+        # Use "system" module path extensions of hierarchical MNS (i.e., w/o module class)
+        return super(CategorizedHMNS, self).det_modpath_extensions(ec)
+
     def det_init_modulepaths(self, ec):
         """
         Determine list of initial module paths (i.e., top of the hierarchy).

easybuild/tools/module_naming_scheme/mns.py

@@ -126,6 +126,18 @@ def det_modpath_extensions(self, ec):
         # by default: an empty list of subdirectories to extend $MODULEPATH with
         return []
 
+    def det_user_modpath_extensions(self, ec):
+        """
+        Determine list of subdirectories relative to the user-specific modules directory for which to extend
+        $MODULEPATH with when this module is loaded (if any).
+
+        @param ec: dict-like object with easyconfig parameter values; for now only the 'name',
+                   'version', 'versionsuffix' and 'toolchain' parameters are guaranteed to be available
+        @return: A list of $MODULEPATH subdirectories.
+        """
+        # by default: use "system" module path extensions of naming scheme
+        return self.det_modpath_extensions(ec)
+
     def det_init_modulepaths(self, ec):
         """
         Determine initial module paths, where the modules that are top of the hierarchy (if any) live.

easybuild/tools/modules.py

@@ -588,7 +588,7 @@ def modpath_extensions_for(self, mod_names):
         modpath_exts = {}
         for mod_name in mod_names:
             modtxt = self.read_module_file(mod_name)
-            useregex = re.compile(r"^\s*module\s+use\s+(\S+)", re.M)
+            useregex = re.compile(r'^\s*module\s+use\s+"?([^"\s]+)"?', re.M)
             exts = useregex.findall(modtxt)
 
             self.log.debug("Found $MODULEPATH extensions for %s: %s" % (mod_name, exts))

easybuild/tools/options.py

@@ -319,6 +319,7 @@ def config_options(self):
             'subdir-modules': ("Installpath subdir for modules", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_modules']),
             'subdir-software': ("Installpath subdir for software",
                                 None, 'store', DEFAULT_PATH_SUBDIRS['subdir_software']),
+            'subdir-user-modules': ("Base path of user-specific modules relative to their $HOME", None, 'store', None),
             'suffix-modules-path': ("Suffix for module files install path", None, 'store', GENERAL_CLASS),
             # this one is sort of an exception, it's something jobscripts can set,
             # has no real meaning for regular eb usage

test/framework/easyblock.py

@@ -166,6 +166,60 @@ def test_fake_module_load(self):
         eb.close_log()
         os.remove(eb.logfile)
 
+    def test_make_module_extend_modpath(self):
+        """Test for make_module_extend_modpath"""
+        self.contents = '\n'.join([
+            'easyblock = "ConfigureMake"',
+            'name = "pi"',
+            'version = "3.14"',
+            'homepage = "http://example.com"',
+            'description = "test easyconfig"',
+            'toolchain = {"name":"dummy", "version": "dummy"}',
+            'moduleclass = "compiler"',
+        ])
+        self.writeEC()
+        eb = EasyBlock(EasyConfig(self.eb_file))
+        eb.installdir = config.install_path()
+
+        # no $MODULEPATH extensions for default module naming scheme (EasyBuildMNS)
+        self.assertEqual(eb.make_module_extend_modpath(), '')
+
+        usermodsdir = 'my/own/modules'
+        modclasses = ['compiler', 'tools']
+        os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedHMNS'
+        build_options = {
+            'subdir_user_modules': usermodsdir,
+            'valid_module_classes': modclasses,
+        }
+        init_config(build_options=build_options)
+        eb = EasyBlock(EasyConfig(self.eb_file))
+        eb.installdir = config.install_path()
+
+        txt = eb.make_module_extend_modpath()
+        if get_module_syntax() == 'Tcl':
+            regexs = [r'^module use ".*/modules/all/Compiler/pi/3.14/%s"$' % c for c in modclasses]
+            home = r'\$env\(HOME\)'
+            regexs.extend([
+                # extension for user modules is guarded
+                r'if { \[ file isdirectory \[ file join %s "%s/Compiler/pi/3.14" \] \] } {$' % (home, usermodsdir),
+                # no per-moduleclass extension for user modules
+                r'^\s+module use \[ file join %s "%s/Compiler/pi/3.14"\ ]$' % (home, usermodsdir),
+            ])
+        elif get_module_syntax() == 'Lua':
+            regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/all/Compiler/pi/3.14/%s"\)$' % c for c in modclasses]
+            home = r'os.getenv\("HOME"\)'
+            regexs.extend([
+                # extension for user modules is guarded
+                r'if isDir\(pathJoin\(%s, "%s/Compiler/pi/3.14"\)\) then' % (home, usermodsdir),
+                # no per-moduleclass extension for user modules
+                r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, "%s/Compiler/pi/3.14"\)\)' % (home, usermodsdir),
+            ])
+        else:
+            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
+        for regex in regexs:
+            regex = re.compile(regex, re.M)
+            self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
+
     def test_make_module_req(self):
         """Testcase for make_module_req"""
         self.contents = '\n'.join([
@@ -764,10 +818,15 @@ def test_exclude_path_to_top_of_module_tree(self):
             modfile_path = os.path.join(modpath, modfile_path)
             modtxt = read_file(modfile_path)
 
-            for imkl_dep in excluded_deps:
-                tup = (imkl_dep, modfile_path, modtxt)
+            for dep in excluded_deps:
+                tup = (dep, modfile_path, modtxt)
                 failmsg = "No 'module load' statement found for '%s' not found in module %s: %s" % tup
-                self.assertFalse(re.search("module load %s" % imkl_dep, modtxt), failmsg)
+                if get_module_syntax() == 'Tcl':
+                    self.assertFalse(re.search('module load %s' % dep, modtxt), failmsg)
+                elif get_module_syntax() == 'Lua':
+                    self.assertFalse(re.search('load("%s")' % dep, modtxt), failmsg)
+                else:
+                    self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
 
         os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme
         init_config(build_options=build_options)