From 9dbf558f156dc78f3b660c105c20443967e58165 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 29 Apr 2020 16:08:58 +0200 Subject: [PATCH] Fix for module-alias on EnvironmentModules and Lmod 6 Deprecate mod_exists_regex_template in favor for better output parsing Output parsing of module show is now more accurate as errors are handled and only first non-whitespace line is checked Usual outputs for non-existant modules: Empty or `(0):ERROR:...` For existing modules the first line (after comments etc) contains the full path of the module file with a colon at the end. --- easybuild/tools/modules.py | 45 +++++++++++++++----------------------- test/framework/modules.py | 22 +++++++++++++------ 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index b86b8fcdf0..18ef97ed3d 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -531,12 +531,12 @@ def module_wrapper_exists(self, mod_name, modulerc_fn='.modulerc', mod_wrapper_r return wrapped_mod - def exist(self, mod_names, mod_exists_regex_template=r'^\s*\S*/%s.*:\s*$', skip_avail=False, maybe_partial=True): + def exist(self, mod_names, mod_exists_regex_template=None, skip_avail=False, maybe_partial=True): """ Check if modules with specified names exists. :param mod_names: list of module names - :param mod_exists_regex_template: template regular expression to search 'module show' output with + :param mod_exists_regex_template: DEPRECATED and unused :param skip_avail: skip checking through 'module avail', only check via 'module show' :param maybe_partial: indicates if the module name may be a partial module name """ @@ -546,22 +546,26 @@ def mod_exists_via_show(mod_name): :param mod_name: module name """ - txt = self.show(mod_name) + stderr = self.show(mod_name) res = False - names_to_check = [mod_name] - # The module might be an alias where the target can be arbitrary - # As a compromise we check for the base name of the module so we find - # "Java/whatever-11" when searching for "Java/11" (--> basename="Java") - basename = os.path.dirname(mod_name) - if basename: - names_to_check.append(basename) - for name in names_to_check: - mod_exists_regex = mod_exists_regex_template % re.escape(name) - if re.search(mod_exists_regex, txt, re.M): - res = True + # Parse the output: + # - Skip whitespace + # - Any error -> Module does not exist + # - Check first non-whitespace line for something that looks like an absolute path terminated by a colon + mod_exists_regex = r'\s*/.+:\s*' + for line in stderr.split('\n'): + if OUTPUT_MATCHES['whitespace'].search(line): + continue + if OUTPUT_MATCHES['error'].search(line): break + if re.match(mod_exists_regex, line): + res = True + break return res + if mod_exists_regex_template is not None: + self.log.deprecated('mod_exists_regex_template is no longer used', '5.0') + if skip_avail: avail_mod_names = [] elif len(mod_names) == 1: @@ -1418,19 +1422,6 @@ def module_wrapper_exists(self, mod_name): return res - def exist(self, mod_names, skip_avail=False, maybe_partial=True): - """ - Check if modules with specified names exists. - - :param mod_names: list of module names - :param skip_avail: skip checking through 'module avail', only check via 'module show' - """ - # module file may be either in Tcl syntax (no file extension) or Lua sytax (.lua extension); - # the current configuration for matters little, since the module may have been installed with a different cfg; - # Lmod may pick up both Tcl and Lua module files, regardless of the EasyBuild configuration - return super(Lmod, self).exist(mod_names, mod_exists_regex_template=r'^\s*\S*/%s.*(\.lua)?:\s*$', - skip_avail=skip_avail, maybe_partial=maybe_partial) - def get_setenv_value_from_modulefile(self, mod_name, var_name): """ Get value for specific 'setenv' statement from module file for the specified module. diff --git a/test/framework/modules.py b/test/framework/modules.py index 1d442e3488..6ede04bdfa 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -279,6 +279,7 @@ def test_exist(self): java_mod_dir = os.path.join(self.test_prefix, 'Java') write_file(os.path.join(java_mod_dir, '1.8.0_181'), '#%Module') + write_file(os.path.join(self.test_prefix, 'toy', '42.1337'), '#%Module') if self.modtool.__class__ == EnvironmentModulesC: modulerc_tcl_txt = '\n'.join([ @@ -289,8 +290,11 @@ def test_exist(self): 'if {"Java/site_default" eq [module-info version Java/site_default]} {', ' module-version Java/1.8.0_181 site_default', '}', - 'if {"JavaAlias" eq [module-info version JavaAlias]} {', - ' module-alias JavaAlias Java/1.8.0_181', + 'if {"Java/Alias" eq [module-info version Java/Alias]} {', + ' module-alias Java/Alias toy/42.1337', + '}', + 'if {"Java/NonExist" eq [module-info version Java/NonExist]} {', + ' module-alias Java/NonExist non_existant/1', '}', ]) else: @@ -298,7 +302,9 @@ def test_exist(self): '#%Module', 'module-version Java/1.8.0_181 1.8', 'module-version Java/1.8.0_181 site_default', - 'module-alias JavaAlias Java/1.8.0_181', + 'module-alias Java/Alias toy/42.1337', + # 'module-alias Java/NonExist non_existant/1', # (only) LMod has this in module avail, disable for now + 'module-alias JavaAlias Java/1.8.0_181', # LMod 7+ only ]) write_file(os.path.join(java_mod_dir, '.modulerc'), modulerc_tcl_txt) @@ -309,13 +315,15 @@ def test_exist(self): self.assertTrue('Java/1.8' in avail_mods) self.assertTrue('Java/site_default' in avail_mods) self.assertTrue('JavaAlias' in avail_mods) + self.assertEqual(self.modtool.exist(['JavaAlias']), [True]) self.assertEqual(self.modtool.exist(['Java/1.8', 'Java/1.8.0_181']), [True, True]) - # check for an alias with a different version suffix than the base module - self.assertEqual(self.modtool.exist(['Java/site_default']), [True]) - # And completely different name - self.assertEqual(self.modtool.exist(['JavaAlias']), [True]) + # Check for aliases: + # - different version suffix than the base module + # - completely different name + # - alias to non existant module + self.assertEqual(self.modtool.exist(['Java/site_default', 'Java/Alias', 'Java/NonExist']), [True, True, False]) reset_module_caches()