From 58c15f45866be39faa1226e480edee929f30f951 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 26 Feb 2020 18:24:03 +0100 Subject: [PATCH 01/16] Also check for module basename in module exist Allows to find Java/whatver-11 from "module show Java/11" --- easybuild/tools/modules.py | 22 ++++++++++++++++++---- test/framework/modules.py | 16 +++++++++++++++- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 8a5323434d..531f8253fb 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -543,9 +543,21 @@ def mod_exists_via_show(mod_name): :param mod_name: module name """ - mod_exists_regex = mod_exists_regex_template % re.escape(mod_name) txt = self.show(mod_name) - return bool(re.search(mod_exists_regex, txt, re.M)) + res = False + names_to_check = [mod_name] + # The module might be an alias where the target can be arbitrary + # As a compromise we check for the base name of the module so we find + # "Java/whatever-11" when searching for "Java/11" (--> basename="Java") + basename = os.path.dirname(mod_name) + if basename: + names_to_check.append(basename) + for name in names_to_check: + mod_exists_regex = mod_exists_regex_template % re.escape(name) + if re.search(mod_exists_regex, txt, re.M): + res = True + break + return res if skip_avail: avail_mod_names = [] @@ -643,7 +655,7 @@ def show(self, mod_name): ans = MODULE_SHOW_CACHE[key] self.log.debug("Found cached result for 'module show %s' with key '%s': %s", mod_name, key, ans) else: - ans = self.run_module('show', mod_name, check_output=False, return_output=True) + ans = self.run_module('show', mod_name, check_output=False, return_stderr=True) MODULE_SHOW_CACHE[key] = ans self.log.debug("Cached result for 'module show %s' with key '%s': %s", mod_name, key, ans) @@ -765,7 +777,9 @@ def run_module(self, *args, **kwargs): if kwargs.get('check_output', True): self.check_module_output(full_cmd, stdout, stderr) - if kwargs.get('return_output', False): + if kwargs.get('return_stderr', False): + return stderr + elif kwargs.get('return_output', False): return stdout + stderr else: # the module command was run with an outdated selected environment variables (see LD_ENV_VAR_KEYS list) diff --git a/test/framework/modules.py b/test/framework/modules.py index 93015b4a07..37a4a82947 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -175,11 +175,15 @@ def test_exist(self): 'if {"Java/1.8" eq [module-info version Java/1.8]} {', ' module-version Java/1.8.0_181 1.8', '}', + 'if {"Java/site_default" eq [module-info version Java/site_default]} {', + ' module-version Java/1.8.0_181 site_default', + '}', ]) else: modulerc_tcl_txt = '\n'.join([ '#%Module', 'module-version Java/1.8.0_181 1.8', + 'module-version Java/1.8.0_181 site_default', ]) write_file(os.path.join(java_mod_dir, '.modulerc'), modulerc_tcl_txt) @@ -189,6 +193,8 @@ def test_exist(self): if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.0'): self.assertTrue('Java/1.8' in avail_mods) self.assertEqual(self.modtool.exist(['Java/1.8', 'Java/1.8.0_181']), [True, True]) + # Check for an alias with a different version suffix than the base module + self.assertEqual(self.modtool.exist(['Java/site_default']), [True]) self.assertEqual(self.modtool.module_wrapper_exists('Java/1.8'), 'Java/1.8.0_181') reset_module_caches() @@ -200,6 +206,7 @@ def test_exist(self): self.assertTrue('Core/Java/1.8.0_181' in self.modtool.available()) self.assertEqual(self.modtool.exist(['Core/Java/1.8.0_181']), [True]) self.assertEqual(self.modtool.exist(['Core/Java/1.8']), [True]) + self.assertEqual(self.modtool.exist(['Core/Java/site_default']), [True]) self.assertEqual(self.modtool.module_wrapper_exists('Core/Java/1.8'), 'Core/Java/1.8.0_181') # also check with .modulerc.lua for Lmod 7.8 or newer @@ -208,12 +215,18 @@ def test_exist(self): reset_module_caches() remove_file(os.path.join(java_mod_dir, '.modulerc')) - write_file(os.path.join(java_mod_dir, '.modulerc.lua'), 'module_version("Java/1.8.0_181", "1.8")') + write_file(os.path.join(java_mod_dir, '.modulerc.lua'), + '\n'.join([ + 'module_version("Java/1.8.0_181", "1.8")', + 'module_version("Java/1.8.0_181", "site_default")', + ])) avail_mods = self.modtool.available() self.assertTrue('Java/1.8.0_181' in avail_mods) self.assertTrue('Java/1.8' in avail_mods) self.assertEqual(self.modtool.exist(['Java/1.8', 'Java/1.8.0_181']), [True, True]) + # Check for an alias with a different version suffix than the base module + self.assertEqual(self.modtool.exist(['Java/site_default']), [True]) self.assertEqual(self.modtool.module_wrapper_exists('Java/1.8'), 'Java/1.8.0_181') reset_module_caches() @@ -223,6 +236,7 @@ def test_exist(self): self.assertTrue('Core/Java/1.8.0_181' in self.modtool.available()) self.assertEqual(self.modtool.exist(['Core/Java/1.8.0_181']), [True]) self.assertEqual(self.modtool.exist(['Core/Java/1.8']), [True]) + self.assertEqual(self.modtool.exist(['Core/Java/site_default']), [True]) self.assertEqual(self.modtool.module_wrapper_exists('Core/Java/1.8'), 'Core/Java/1.8.0_181') def test_load(self): From 5169fe12ae9497eb38cff96e04d4dd558b88ad93 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 10 Apr 2020 09:13:50 +0200 Subject: [PATCH 02/16] add tests for ModulesTool.show and ModulesTool.run_module --- test/framework/modules.py | 121 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 116 insertions(+), 5 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 37a4a82947..492d91f6e3 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -44,6 +44,7 @@ from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig.easyconfig import EasyConfig from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.environment import modify_env from easybuild.tools.filetools import adjust_permissions, copy_file, copy_dir, mkdir from easybuild.tools.filetools import read_file, remove_dir, remove_file, symlink, write_file from easybuild.tools.modules import EnvironmentModules, EnvironmentModulesC, EnvironmentModulesTcl, Lmod, NoModulesTool @@ -92,6 +93,87 @@ def test_long_module_path(self): shutil.rmtree(tmpdir) + def test_run_module(self): + """Test for ModulesTool.run_module method.""" + + testdir = os.path.dirname(os.path.abspath(__file__)) + + for key in ['EBROOTGCC', 'EBROOTOPENMPI', 'EBROOTOPENBLAS']: + if key in os.environ: + del os.environ[key] + + # arguments can be passed in two ways: multiple arguments, or just 1 list argument + self.modtool.run_module('load', 'GCC/6.4.0-2.28') + self.assertEqual(os.environ['EBROOTGCC'], '/prefix/software/GCC/6.4.0-2.28') + + # restore original environment + modify_env(os.environ, self.orig_environ, verbose=False) + self.reset_modulepath([os.path.join(testdir, 'modules')]) + + self.assertFalse('EBROOTGCC' in os.environ) + self.modtool.run_module(['load', 'GCC/6.4.0-2.28']) + self.assertEqual(os.environ['EBROOTGCC'], '/prefix/software/GCC/6.4.0-2.28') + + # by default, exit code is checked and an error is raised if we run something that fails + error_pattern = "Module command 'module thisdoesnotmakesense' failed with exit code [1-9]" + self.assertErrorRegex(EasyBuildError, error_pattern, self.modtool.run_module, 'thisdoesnotmakesense') + + error_pattern = "Module command 'module load nosuchmodule/1.2.3' failed with exit code [1-9]" + self.assertErrorRegex(EasyBuildError, error_pattern, self.modtool.run_module, 'load', 'nosuchmodule/1.2.3') + + # we can choose to blatently ignore the exit code, + # and also disable the output check that serves as a fallback + self.modtool.run_module('thisdoesnotmakesense', check_exit_code=False, check_output=False) + self.modtool.run_module('load', 'nosuchmodule/1.2.3', check_exit_code=False, check_output=False) + + # by default, the output (stdout+stderr) produced by the command is processed; + # result is a list of useful info (module names in case of list/avail) + res = self.modtool.run_module('list') + self.assertEqual(res, [{'mod_name': 'GCC/6.4.0-2.28', 'default': None}]) + + res = self.modtool.run_module('avail', 'GCC/4.6') + self.assertTrue(isinstance(res, list)) + self.assertEqual(sorted([x['mod_name'] for x in res]), ['GCC/4.6.3', 'GCC/4.6.4']) + + # loading a module produces no output, so we get an empty list + res = self.modtool.run_module('load', 'OpenMPI/2.1.2-GCC-6.4.0-2.28') + self.assertEqual(res, []) + self.assertEqual(os.environ['EBROOTOPENMPI'], '/prefix/software/OpenMPI/2.1.2-GCC-6.4.0-2.28') + + # we can opt into getting back the raw output (stdout + stderr); + # in that cases, the output includes Python statements to change the environment; + # the changes that would be made by the module command are *not* applied to the environment + out = self.modtool.run_module('load', 'OpenBLAS/0.2.20-GCC-6.4.0-2.28', return_output=True) + patterns = [ + r"^os.environ\[.EBROOTOPENBLAS.\]\s*=\s*./prefix/software/OpenBLAS/0.2.20-GCC-6.4.0-2.28.", + r"^os.environ\[.LOADEDMODULES.\]\s*=.*OpenBLAS/0.2.20-GCC-6.4.0-2.28", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(out), "Pattern '%s' should be found in: %s" % (regex.pattern, out)) + + # OpenBLAS module did *not* get loaded + self.assertFalse('EBROOTOPENBLAS' in os.environ) + res = self.modtool.list() + expected = ['GCC/6.4.0-2.28', 'OpenMPI/2.1.2-GCC-6.4.0-2.28', 'hwloc/1.11.8-GCC-6.4.0-2.28'] + self.assertEqual(sorted([x['mod_name'] for x in res]), expected) + + # we can also only obtain the stderr output (which contains the user-facing output), + # and just drop the stdout output (which contains the statements to change the environment) + out = self.modtool.run_module('show', 'OpenBLAS/0.2.20-GCC-6.4.0-2.28', return_stderr=True) + patterns = [ + r"test/framework/modules/OpenBLAS/0.2.20-GCC-6.4.0-2.28:\s*$", + r"setenv\W+EBROOTOPENBLAS.+/prefix/software/OpenBLAS/0.2.20-GCC-6.4.0-2.28", + r"prepend[_-]path\W+LD_LIBRARY_PATH.+/prefix/software/OpenBLAS/0.2.20-GCC-6.4.0-2.28/lib", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(out), "Pattern '%s' should be found in: %s" % (regex.pattern, out)) + + # show method only returns user-facing output (obtained via stderr), not changes to the environment + regex = re.compile(r'^os\.environ\[', re.M) + self.assertFalse(regex.search(out), "Pattern '%s' should not be found in: %s" % (regex.pattern, out)) + def test_avail(self): """Test if getting a (restricted) list of available modules works.""" self.init_testmods() @@ -192,10 +274,15 @@ def test_exist(self): self.assertTrue('Java/1.8.0_181' in avail_mods) if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.0'): self.assertTrue('Java/1.8' in avail_mods) + self.assertTrue('Java/site_default' in avail_mods) + self.assertEqual(self.modtool.exist(['Java/1.8', 'Java/1.8.0_181']), [True, True]) - # Check for an alias with a different version suffix than the base module + + # check for an alias with a different version suffix than the base module self.assertEqual(self.modtool.exist(['Java/site_default']), [True]) + self.assertEqual(self.modtool.module_wrapper_exists('Java/1.8'), 'Java/1.8.0_181') + self.assertEqual(self.modtool.module_wrapper_exists('Java/site_default'), 'Java/1.8.0_181') reset_module_caches() @@ -208,6 +295,7 @@ def test_exist(self): self.assertEqual(self.modtool.exist(['Core/Java/1.8']), [True]) self.assertEqual(self.modtool.exist(['Core/Java/site_default']), [True]) self.assertEqual(self.modtool.module_wrapper_exists('Core/Java/1.8'), 'Core/Java/1.8.0_181') + self.assertEqual(self.modtool.module_wrapper_exists('Core/Java/site_default'), 'Core/Java/1.8.0_181') # also check with .modulerc.lua for Lmod 7.8 or newer if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.8'): @@ -225,9 +313,12 @@ def test_exist(self): self.assertTrue('Java/1.8.0_181' in avail_mods) self.assertTrue('Java/1.8' in avail_mods) self.assertEqual(self.modtool.exist(['Java/1.8', 'Java/1.8.0_181']), [True, True]) - # Check for an alias with a different version suffix than the base module + + # check for an alias with a different version suffix than the base module self.assertEqual(self.modtool.exist(['Java/site_default']), [True]) + self.assertEqual(self.modtool.module_wrapper_exists('Java/1.8'), 'Java/1.8.0_181') + self.assertEqual(self.modtool.module_wrapper_exists('Java/site_default'), 'Java/1.8.0_181') reset_module_caches() @@ -238,6 +329,7 @@ def test_exist(self): self.assertEqual(self.modtool.exist(['Core/Java/1.8']), [True]) self.assertEqual(self.modtool.exist(['Core/Java/site_default']), [True]) self.assertEqual(self.modtool.module_wrapper_exists('Core/Java/1.8'), 'Core/Java/1.8.0_181') + self.assertEqual(self.modtool.module_wrapper_exists('Core/Java/site_default'), 'Core/Java/1.8.0_181') def test_load(self): """ test if we load one module it is in the loaded_modules """ @@ -298,6 +390,25 @@ def test_load(self): self.assertEqual(os.environ.get('EBROOTGCC'), None) self.assertFalse(loaded_modules[-1] == 'GCC/6.4.0-2.28') + def test_show(self): + """Test for ModulesTool.show method.""" + + out = self.modtool.show('GCC/7.3.0-2.30') + + patterns = [ + # full path to module is included in output of 'show' + r"test/framework/modules/GCC/7.3.0-2.30:\s*$", + r"setenv\W+EBROOTGCC.+prefix/software/GCC/7.3.0-2.30", + r"^prepend[_-]path\W+PATH.+/prefix/software/GCC/7.3.0-2.30/bin", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(out), "Pattern '%s' should be found in: %s" % (regex.pattern, out)) + + # show method only returns user-facing output (obtained via stderr), not changes to the environment + regex = re.compile(r'^os\.environ\[', re.M) + self.assertFalse(regex.search(out), "Pattern '%s' should not be found in: %s" % (regex.pattern, out)) + def test_curr_module_paths(self): """Test for curr_module_paths function.""" @@ -905,7 +1016,7 @@ def test_modules_tool_stateless(self): # exact error message depends on Lmod version load_err_msg = '|'.join([ r'These[\s\sn]*module\(s\)[\s\sn]*exist[\s\sn]*but[\s\sn]*cannot[\s\sn]*be', - 'The[\s\sn]*following[\s\sn]*module\(s\)[\s\sn]*are[\s\sn]*unknown', + r'The[\s\sn]*following[\s\sn]*module\(s\)[\s\sn]*are[\s\sn]*unknown', ]) else: load_err_msg = "Unable to locate a modulefile" @@ -1115,7 +1226,7 @@ def check_loaded_modules(): r"^\* GCC/6.4.0-2.28", r"^\* hwloc/1.11.8-GCC-6.4.0-2.28", r"^\* OpenMPI/2.1.2-GCC-6.4.0-2.28", - "This is not recommended since it may affect the installation procedure\(s\) performed by EasyBuild.", + r"This is not recommended since it may affect the installation procedure\(s\) performed by EasyBuild.", "To make EasyBuild allow particular loaded modules, use the --allow-loaded-modules configuration option.", "To specify action to take when loaded modules are detected, use " "--detect-loaded-modules={error,ignore,purge,unload,warn}", @@ -1133,7 +1244,7 @@ def check_loaded_modules(): # error mentioning 1 non-allowed module (OpenMPI), both GCC and hwloc loaded modules are allowed error_pattern = r"Found one or more non-allowed loaded .* module.*\n" - error_pattern += "\* OpenMPI/2.1.2-GCC-6.4.0-2.28\n\nThis is not" + error_pattern += r"\* OpenMPI/2.1.2-GCC-6.4.0-2.28\n\nThis is not" self.assertErrorRegex(EasyBuildError, error_pattern, self.modtool.check_loaded_modules) # check for warning message when purge is being run on loaded modules From 47292c9bfa42c3db87cdca5267b7a94cadf2a5bd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 10 Apr 2020 10:59:07 +0200 Subject: [PATCH 03/16] make test_run_module a bit less strict, to take into account differences between different module tools... --- test/framework/modules.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 492d91f6e3..28d99c4781 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -114,17 +114,34 @@ def test_run_module(self): self.modtool.run_module(['load', 'GCC/6.4.0-2.28']) self.assertEqual(os.environ['EBROOTGCC'], '/prefix/software/GCC/6.4.0-2.28') - # by default, exit code is checked and an error is raised if we run something that fails - error_pattern = "Module command 'module thisdoesnotmakesense' failed with exit code [1-9]" - self.assertErrorRegex(EasyBuildError, error_pattern, self.modtool.run_module, 'thisdoesnotmakesense') - - error_pattern = "Module command 'module load nosuchmodule/1.2.3' failed with exit code [1-9]" - self.assertErrorRegex(EasyBuildError, error_pattern, self.modtool.run_module, 'load', 'nosuchmodule/1.2.3') + # skip tests that rely on exit codes when using EnvironmentModulesTcl modules tool, + # because it doesn't use proper exit codes + if not isinstance(self.modtool, EnvironmentModulesTcl): + + # by default, exit code is checked and an error is raised if we run something that fails + error_pattern = "Module command '.*thisdoesnotmakesense' failed with exit code [1-9]" + self.assertErrorRegex(EasyBuildError, error_pattern, self.modtool.run_module, 'thisdoesnotmakesense') + + # we need to use a different error pattern here with EnvironmentModulesC, + # because a load of a non-existing module doesnt' trigger a non-zero exit code... + # it will still fail though, just differently + if isinstance(self.modtool, EnvironmentModulesC): + error_pattern = "Unable to locate a modulefile for 'nosuchmodule/1.2.3'" + else: + error_pattern = "Module command '.*load nosuchmodule/1.2.3' failed with exit code [1-9]" + self.assertErrorRegex(EasyBuildError, error_pattern, self.modtool.run_module, 'load', 'nosuchmodule/1.2.3') # we can choose to blatently ignore the exit code, - # and also disable the output check that serves as a fallback - self.modtool.run_module('thisdoesnotmakesense', check_exit_code=False, check_output=False) - self.modtool.run_module('load', 'nosuchmodule/1.2.3', check_exit_code=False, check_output=False) + # and also disable the output check that serves as a fallback; + # we also enable return_output here, because trying to apply the environment changes produced + # by a faulty command is bound to cause trouble... + kwargs = { + 'check_exit_code': False, + 'check_output': False, + 'return_output': True, + } + self.modtool.run_module('thisdoesnotmakesense', **kwargs) + self.modtool.run_module('load', 'nosuchmodule/1.2.3', **kwargs) # by default, the output (stdout+stderr) produced by the command is processed; # result is a list of useful info (module names in case of list/avail) @@ -1185,7 +1202,7 @@ def test_load_in_hierarchy(self): def test_exit_code_check(self): """Verify that EasyBuild checks exit code of executed module commands""" if isinstance(self.modtool, Lmod): - error_pattern = "Module command 'module load nosuchmoduleavailableanywhere' failed with exit code" + error_pattern = "Module command '.*load nosuchmoduleavailableanywhere' failed with exit code" else: # Tcl implementations exit with 0 even when a non-existing module is loaded... error_pattern = "Unable to locate a modulefile for 'nosuchmoduleavailableanywhere'" From a7ede60bbc82dea0ec73eb4b43f4426921e12f2a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 10 Apr 2020 10:59:34 +0200 Subject: [PATCH 04/16] mention full executed command in ModulesTool.run_module when command return non-zero exit code --- easybuild/tools/modules.py | 4 ++-- test/framework/toy_build.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 531f8253fb..39bfdf3a15 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -771,8 +771,8 @@ def run_module(self, *args, **kwargs): # also catch and check exit code exit_code = proc.returncode if kwargs.get('check_exit_code', True) and exit_code != 0: - raise EasyBuildError("Module command 'module %s' failed with exit code %s; stderr: %s; stdout: %s", - ' '.join(cmd_list[2:]), exit_code, stderr, stdout) + raise EasyBuildError("Module command '%s' failed with exit code %s; stderr: %s; stdout: %s", + ' '.join(cmd_list), exit_code, stderr, stdout) if kwargs.get('check_output', True): self.check_module_output(full_cmd, stdout, stderr) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index eb660a438b..e8361c738b 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1362,7 +1362,7 @@ def test_external_dependencies(self): write_file(toy_ec, ectxt + extraectxt) if isinstance(self.modtool, Lmod): - err_msg = r"Module command \\'module load nosuchbuilddep/0.0.0\\' failed" + err_msg = r"Module command \\'.*load nosuchbuilddep/0.0.0\\' failed" else: err_msg = r"Unable to locate a modulefile for 'nosuchbuilddep/0.0.0'" @@ -1374,7 +1374,7 @@ def test_external_dependencies(self): write_file(toy_ec, ectxt + extraectxt) if isinstance(self.modtool, Lmod): - err_msg = r"Module command \\'module load nosuchmodule/1.2.3\\' failed" + err_msg = r"Module command \\'.*load nosuchmodule/1.2.3\\' failed" else: err_msg = r"Unable to locate a modulefile for 'nosuchmodule/1.2.3'" From a2cae664db76cc3a660177e021370f881ba166d2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 10 Apr 2020 11:35:16 +0200 Subject: [PATCH 05/16] use full module name in check for ModulesTool.run_module('avail'), to avoid failing test with EnvironmentModulesTcl --- test/framework/modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 28d99c4781..50be378b2c 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -148,9 +148,9 @@ def test_run_module(self): res = self.modtool.run_module('list') self.assertEqual(res, [{'mod_name': 'GCC/6.4.0-2.28', 'default': None}]) - res = self.modtool.run_module('avail', 'GCC/4.6') + res = self.modtool.run_module('avail', 'GCC/4.6.3') self.assertTrue(isinstance(res, list)) - self.assertEqual(sorted([x['mod_name'] for x in res]), ['GCC/4.6.3', 'GCC/4.6.4']) + self.assertEqual(sorted([x['mod_name'] for x in res]), ['GCC/4.6.3']) # loading a module produces no output, so we get an empty list res = self.modtool.run_module('load', 'OpenMPI/2.1.2-GCC-6.4.0-2.28') From a967058833ec2575e5f4b8dc2cbe989d1c0b1ef9 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 14 Apr 2020 12:12:46 +0200 Subject: [PATCH 06/16] Add test for available() with module_alias command --- test/framework/modules.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/framework/modules.py b/test/framework/modules.py index 50be378b2c..f67f37ea6e 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -277,12 +277,16 @@ def test_exist(self): 'if {"Java/site_default" eq [module-info version Java/site_default]} {', ' module-version Java/1.8.0_181 site_default', '}', + 'if {"JavaAlias" eq [module-info version JavaAlias]} {', + ' module-alias JavaAlias Java/1.8.0_181', + '}', ]) else: modulerc_tcl_txt = '\n'.join([ '#%Module', 'module-version Java/1.8.0_181 1.8', 'module-version Java/1.8.0_181 site_default', + 'module-alias JavaAlias Java/1.8.0_181', ]) write_file(os.path.join(java_mod_dir, '.modulerc'), modulerc_tcl_txt) @@ -292,11 +296,14 @@ def test_exist(self): if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.0'): self.assertTrue('Java/1.8' in avail_mods) self.assertTrue('Java/site_default' in avail_mods) + self.assertTrue('JavaAlias' in avail_mods) self.assertEqual(self.modtool.exist(['Java/1.8', 'Java/1.8.0_181']), [True, True]) # check for an alias with a different version suffix than the base module self.assertEqual(self.modtool.exist(['Java/site_default']), [True]) + # And completely different name + self.assertEqual(self.modtool.exist(['JavaAlias']), [True]) self.assertEqual(self.modtool.module_wrapper_exists('Java/1.8'), 'Java/1.8.0_181') self.assertEqual(self.modtool.module_wrapper_exists('Java/site_default'), 'Java/1.8.0_181') @@ -324,6 +331,7 @@ def test_exist(self): '\n'.join([ 'module_version("Java/1.8.0_181", "1.8")', 'module_version("Java/1.8.0_181", "site_default")', + 'module_alias("JavaAlias", "Java/1.8")', ])) avail_mods = self.modtool.available() @@ -333,6 +341,8 @@ def test_exist(self): # check for an alias with a different version suffix than the base module self.assertEqual(self.modtool.exist(['Java/site_default']), [True]) + # And completely different name + self.assertEqual(self.modtool.exist(['JavaAlias']), [True]) self.assertEqual(self.modtool.module_wrapper_exists('Java/1.8'), 'Java/1.8.0_181') self.assertEqual(self.modtool.module_wrapper_exists('Java/site_default'), 'Java/1.8.0_181') From 0877c51657395a78a8772aef4b0c7a099f066a1b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 14 Apr 2020 12:53:57 +0200 Subject: [PATCH 07/16] Add test with modulerc in HOME --- test/framework/modules.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/framework/modules.py b/test/framework/modules.py index f67f37ea6e..15fa72095e 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -37,6 +37,7 @@ import stat import sys from distutils.version import StrictVersion +from contextlib import contextmanager from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -57,6 +58,17 @@ TEST_MODULES_COUNT = 81 +@contextmanager +def temporary_home_dir(): + tmpdir = tempfile.mkdtemp() + orig_home = os.environ['HOME'] + os.environ['HOME'] = tmpdir + try: + yield tmpdir + finally: + os.environ['HOME'] = orig_home + shutil.rmtree(tmpdir) + class ModulesTest(EnhancedTestCase): """Test cases for modules.""" @@ -358,6 +370,21 @@ def test_exist(self): self.assertEqual(self.modtool.module_wrapper_exists('Core/Java/1.8'), 'Core/Java/1.8.0_181') self.assertEqual(self.modtool.module_wrapper_exists('Core/Java/site_default'), 'Core/Java/1.8.0_181') + # Test alias in home directory .modulerc + if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.0'): + # Required or temporary HOME would be in MODULEPATH already + self.init_testmods() + # Sanity check: Module aliases don't exist yet + self.assertEqual(self.modtool.exist(['OpenMPI/99', 'OpenMPIAlias']), [False, False]) + with temporary_home_dir() as home_dir: + reset_module_caches() + write_file(os.path.join(home_dir, '.modulerc'), '\n'.join([ + '#%Module', + 'module-version OpenMPI/2.1.2-GCC-6.4.0-2.28 99', + 'module-alias OpenMPIAlias OpenMPI/2.1.2-GCC-6.4.0-2.28', + ])) + self.assertEqual(self.modtool.exist(['OpenMPI/99', 'OpenMPIAlias']), [True, True]) + def test_load(self): """ test if we load one module it is in the loaded_modules """ self.init_testmods() From 1676ae0d38cf0c9132c73938d455ab8599aa833c Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 14 Apr 2020 13:56:29 +0200 Subject: [PATCH 08/16] Add disallow_deprecated_behaviour to invert allow_deprecated_behaviour --- test/framework/utilities.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 2c2bd73ffb..1d98dbcced 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -125,9 +125,8 @@ def setUp(self): os.environ['EASYBUILD_ROBOT_PATHS'] = os.path.join(testdir, 'easyconfigs', 'test_ecs') # make sure no deprecated behaviour is being triggered (unless intended by the test) - # trip *all* log.deprecated statements by setting deprecation version ridiculously high self.orig_current_version = eb_build_log.CURRENT_VERSION - os.environ['EASYBUILD_DEPRECATED'] = '10000000' + self.disallow_deprecated_behaviour() init_config() @@ -181,6 +180,11 @@ def setUp(self): self.reset_modulepath([os.path.join(testdir, 'modules')]) reset_module_caches() + def disallow_deprecated_behaviour(self): + """trip *all* log.deprecated statements by setting deprecation version ridiculously high""" + os.environ['EASYBUILD_DEPRECATED'] = '10000000' + eb_build_log.CURRENT_VERSION = os.environ['EASYBUILD_DEPRECATED'] + def allow_deprecated_behaviour(self): """Restore EasyBuild version to what it was originally, to allow triggering deprecated behaviour.""" if 'EASYBUILD_DEPRECATED' in os.environ: From 4e1f28672e4728fb023bea6244471e121d02cd4d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 14 Apr 2020 13:56:49 +0200 Subject: [PATCH 09/16] Deprecated module_wrapper_exists and remove its usage from exists --- easybuild/tools/modules.py | 14 ++++---------- test/framework/modules.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 39bfdf3a15..b86b8fcdf0 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -487,7 +487,10 @@ def module_wrapper_exists(self, mod_name, modulerc_fn='.modulerc', mod_wrapper_r """ Determine whether a module wrapper with specified name exists. Only .modulerc file in Tcl syntax is considered here. + DEPRECATED. Use exists() """ + self.log.deprecated('module_wrapper_exists is unreliable and should no longer be used', '5.0') + if mod_wrapper_regex_template is None: mod_wrapper_regex_template = "^[ ]*module-version (?P[^ ]*) %s$" @@ -582,15 +585,6 @@ def mod_exists_via_show(mod_name): self.log.debug("checking whether hidden module %s exists via 'show'..." % mod_name) mod_exists = mod_exists_via_show(mod_name) - # if no module file was found, check whether specified module name can be a 'wrapper' module... - if not mod_exists: - self.log.debug("Module %s not found via module avail/show, checking whether it is a wrapper", mod_name) - wrapped_mod = self.module_wrapper_exists(mod_name) - if wrapped_mod is not None: - # module wrapper only really exists if the wrapped module file is also available - mod_exists = wrapped_mod in avail_mod_names or mod_exists_via_show(wrapped_mod) - self.log.debug("Result for existence check of wrapped module %s: %s", wrapped_mod, mod_exists) - self.log.debug("Result for existence check of %s module: %s", mod_name, mod_exists) mods_exist.append(mod_exists) @@ -1408,7 +1402,7 @@ def prepend_module_path(self, path, set_mod_paths=True, priority=None): def module_wrapper_exists(self, mod_name): """ Determine whether a module wrapper with specified name exists. - First check for wrapper defined in .modulerc.lua, fall back to also checking .modulerc (Tcl syntax). + DEPRECATED. Use exists() """ res = None diff --git a/test/framework/modules.py b/test/framework/modules.py index 15fa72095e..94a790eb76 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -317,8 +317,11 @@ def test_exist(self): # And completely different name self.assertEqual(self.modtool.exist(['JavaAlias']), [True]) + # Allow for now... + self.allow_deprecated_behaviour() self.assertEqual(self.modtool.module_wrapper_exists('Java/1.8'), 'Java/1.8.0_181') self.assertEqual(self.modtool.module_wrapper_exists('Java/site_default'), 'Java/1.8.0_181') + self.disallow_deprecated_behaviour() reset_module_caches() @@ -330,8 +333,11 @@ def test_exist(self): self.assertEqual(self.modtool.exist(['Core/Java/1.8.0_181']), [True]) self.assertEqual(self.modtool.exist(['Core/Java/1.8']), [True]) self.assertEqual(self.modtool.exist(['Core/Java/site_default']), [True]) + + self.allow_deprecated_behaviour() self.assertEqual(self.modtool.module_wrapper_exists('Core/Java/1.8'), 'Core/Java/1.8.0_181') self.assertEqual(self.modtool.module_wrapper_exists('Core/Java/site_default'), 'Core/Java/1.8.0_181') + self.disallow_deprecated_behaviour() # also check with .modulerc.lua for Lmod 7.8 or newer if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.8'): @@ -356,8 +362,10 @@ def test_exist(self): # And completely different name self.assertEqual(self.modtool.exist(['JavaAlias']), [True]) + self.allow_deprecated_behaviour() self.assertEqual(self.modtool.module_wrapper_exists('Java/1.8'), 'Java/1.8.0_181') self.assertEqual(self.modtool.module_wrapper_exists('Java/site_default'), 'Java/1.8.0_181') + self.disallow_deprecated_behaviour() reset_module_caches() @@ -367,8 +375,11 @@ def test_exist(self): self.assertEqual(self.modtool.exist(['Core/Java/1.8.0_181']), [True]) self.assertEqual(self.modtool.exist(['Core/Java/1.8']), [True]) self.assertEqual(self.modtool.exist(['Core/Java/site_default']), [True]) + + self.allow_deprecated_behaviour() self.assertEqual(self.modtool.module_wrapper_exists('Core/Java/1.8'), 'Core/Java/1.8.0_181') self.assertEqual(self.modtool.module_wrapper_exists('Core/Java/site_default'), 'Core/Java/1.8.0_181') + self.disallow_deprecated_behaviour() # Test alias in home directory .modulerc if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.0'): From e4566a62d856d22f3c104f08a672fae028658f6d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 29 Apr 2020 12:44:57 +0200 Subject: [PATCH 10/16] Remove tests for deprecated module_wrapper_exists --- test/framework/modules.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 94a790eb76..1720e8ef64 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -317,12 +317,6 @@ def test_exist(self): # And completely different name self.assertEqual(self.modtool.exist(['JavaAlias']), [True]) - # Allow for now... - self.allow_deprecated_behaviour() - self.assertEqual(self.modtool.module_wrapper_exists('Java/1.8'), 'Java/1.8.0_181') - self.assertEqual(self.modtool.module_wrapper_exists('Java/site_default'), 'Java/1.8.0_181') - self.disallow_deprecated_behaviour() - reset_module_caches() # what if we're in an HMNS setting... @@ -334,11 +328,6 @@ def test_exist(self): self.assertEqual(self.modtool.exist(['Core/Java/1.8']), [True]) self.assertEqual(self.modtool.exist(['Core/Java/site_default']), [True]) - self.allow_deprecated_behaviour() - self.assertEqual(self.modtool.module_wrapper_exists('Core/Java/1.8'), 'Core/Java/1.8.0_181') - self.assertEqual(self.modtool.module_wrapper_exists('Core/Java/site_default'), 'Core/Java/1.8.0_181') - self.disallow_deprecated_behaviour() - # also check with .modulerc.lua for Lmod 7.8 or newer if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.8'): shutil.move(os.path.join(self.test_prefix, 'Core', 'Java'), java_mod_dir) @@ -362,11 +351,6 @@ def test_exist(self): # And completely different name self.assertEqual(self.modtool.exist(['JavaAlias']), [True]) - self.allow_deprecated_behaviour() - self.assertEqual(self.modtool.module_wrapper_exists('Java/1.8'), 'Java/1.8.0_181') - self.assertEqual(self.modtool.module_wrapper_exists('Java/site_default'), 'Java/1.8.0_181') - self.disallow_deprecated_behaviour() - reset_module_caches() # back to HMNS setup @@ -376,11 +360,6 @@ def test_exist(self): self.assertEqual(self.modtool.exist(['Core/Java/1.8']), [True]) self.assertEqual(self.modtool.exist(['Core/Java/site_default']), [True]) - self.allow_deprecated_behaviour() - self.assertEqual(self.modtool.module_wrapper_exists('Core/Java/1.8'), 'Core/Java/1.8.0_181') - self.assertEqual(self.modtool.module_wrapper_exists('Core/Java/site_default'), 'Core/Java/1.8.0_181') - self.disallow_deprecated_behaviour() - # Test alias in home directory .modulerc if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.0'): # Required or temporary HOME would be in MODULEPATH already From 3c4aa43d78075281bd6ac2259a92b70d1ec75097 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 30 Apr 2020 09:04:50 +0200 Subject: [PATCH 11/16] Fix failing exist HMNS tests on LMod and EnvironmentModulesTcl For HMNS the .modulerc needs updating on EnvironmentModules however the old fallback wrongly considered the current version as correct (depending on the actual location of the .modulerc) and returned True even though the module is not usable --- test/framework/modules.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 1720e8ef64..a2a70ec6f6 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -325,8 +325,12 @@ def test_exist(self): self.assertTrue('Core/Java/1.8.0_181' in self.modtool.available()) self.assertEqual(self.modtool.exist(['Core/Java/1.8.0_181']), [True]) - self.assertEqual(self.modtool.exist(['Core/Java/1.8']), [True]) - self.assertEqual(self.modtool.exist(['Core/Java/site_default']), [True]) + # module-version only works for EnvironmentModules(C) as LMod and EnvironmentModulesTcl would need updating + # to full path, see https://github.com/TACC/Lmod/issues/446 + if isinstance(self.modtool, Lmod) or self.modtool.__class__ == EnvironmentModulesTcl: + self.assertEqual(self.modtool.exist(['Core/Java/1.8', 'Core/Java/site_default']), [False, False]) + else: + self.assertEqual(self.modtool.exist(['Core/Java/1.8', 'Core/Java/site_default']), [True, True]) # also check with .modulerc.lua for Lmod 7.8 or newer if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.8'): @@ -357,8 +361,8 @@ def test_exist(self): shutil.move(java_mod_dir, os.path.join(self.test_prefix, 'Core', 'Java')) self.assertTrue('Core/Java/1.8.0_181' in self.modtool.available()) self.assertEqual(self.modtool.exist(['Core/Java/1.8.0_181']), [True]) - self.assertEqual(self.modtool.exist(['Core/Java/1.8']), [True]) - self.assertEqual(self.modtool.exist(['Core/Java/site_default']), [True]) + self.assertEqual(self.modtool.exist(['Core/Java/1.8']), [False]) + self.assertEqual(self.modtool.exist(['Core/Java/site_default']), [False]) # Test alias in home directory .modulerc if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.0'): From 5be85308f5643951f6127ea8a5691686bd299d0e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 29 Apr 2020 16:08:58 +0200 Subject: [PATCH 12/16] Fix for module-alias on EnvironmentModules and Lmod 6 Deprecate mod_exists_regex_template in favor for better output parsing Output parsing of module show is now more accurate as errors are handled and only first non-whitespace line is checked Usual outputs for non-existant modules: Empty or `(0):ERROR:...` For existing modules the first line (after comments etc) contains the full path of the module file with a colon at the end. --- easybuild/tools/modules.py | 45 +++++++++++++++----------------------- test/framework/modules.py | 19 ++++++++++------ 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index b86b8fcdf0..18ef97ed3d 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -531,12 +531,12 @@ def module_wrapper_exists(self, mod_name, modulerc_fn='.modulerc', mod_wrapper_r return wrapped_mod - def exist(self, mod_names, mod_exists_regex_template=r'^\s*\S*/%s.*:\s*$', skip_avail=False, maybe_partial=True): + def exist(self, mod_names, mod_exists_regex_template=None, skip_avail=False, maybe_partial=True): """ Check if modules with specified names exists. :param mod_names: list of module names - :param mod_exists_regex_template: template regular expression to search 'module show' output with + :param mod_exists_regex_template: DEPRECATED and unused :param skip_avail: skip checking through 'module avail', only check via 'module show' :param maybe_partial: indicates if the module name may be a partial module name """ @@ -546,22 +546,26 @@ def mod_exists_via_show(mod_name): :param mod_name: module name """ - txt = self.show(mod_name) + stderr = self.show(mod_name) res = False - names_to_check = [mod_name] - # The module might be an alias where the target can be arbitrary - # As a compromise we check for the base name of the module so we find - # "Java/whatever-11" when searching for "Java/11" (--> basename="Java") - basename = os.path.dirname(mod_name) - if basename: - names_to_check.append(basename) - for name in names_to_check: - mod_exists_regex = mod_exists_regex_template % re.escape(name) - if re.search(mod_exists_regex, txt, re.M): - res = True + # Parse the output: + # - Skip whitespace + # - Any error -> Module does not exist + # - Check first non-whitespace line for something that looks like an absolute path terminated by a colon + mod_exists_regex = r'\s*/.+:\s*' + for line in stderr.split('\n'): + if OUTPUT_MATCHES['whitespace'].search(line): + continue + if OUTPUT_MATCHES['error'].search(line): break + if re.match(mod_exists_regex, line): + res = True + break return res + if mod_exists_regex_template is not None: + self.log.deprecated('mod_exists_regex_template is no longer used', '5.0') + if skip_avail: avail_mod_names = [] elif len(mod_names) == 1: @@ -1418,19 +1422,6 @@ def module_wrapper_exists(self, mod_name): return res - def exist(self, mod_names, skip_avail=False, maybe_partial=True): - """ - Check if modules with specified names exists. - - :param mod_names: list of module names - :param skip_avail: skip checking through 'module avail', only check via 'module show' - """ - # module file may be either in Tcl syntax (no file extension) or Lua sytax (.lua extension); - # the current configuration for matters little, since the module may have been installed with a different cfg; - # Lmod may pick up both Tcl and Lua module files, regardless of the EasyBuild configuration - return super(Lmod, self).exist(mod_names, mod_exists_regex_template=r'^\s*\S*/%s.*(\.lua)?:\s*$', - skip_avail=skip_avail, maybe_partial=maybe_partial) - def get_setenv_value_from_modulefile(self, mod_name, var_name): """ Get value for specific 'setenv' statement from module file for the specified module. diff --git a/test/framework/modules.py b/test/framework/modules.py index a2a70ec6f6..0c269748ae 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -279,6 +279,7 @@ def test_exist(self): java_mod_dir = os.path.join(self.test_prefix, 'Java') write_file(os.path.join(java_mod_dir, '1.8.0_181'), '#%Module') + write_file(os.path.join(self.test_prefix, 'toy', '42.1337'), '#%Module') if self.modtool.__class__ == EnvironmentModulesC: modulerc_tcl_txt = '\n'.join([ @@ -289,16 +290,15 @@ def test_exist(self): 'if {"Java/site_default" eq [module-info version Java/site_default]} {', ' module-version Java/1.8.0_181 site_default', '}', - 'if {"JavaAlias" eq [module-info version JavaAlias]} {', - ' module-alias JavaAlias Java/1.8.0_181', - '}', ]) else: modulerc_tcl_txt = '\n'.join([ '#%Module', 'module-version Java/1.8.0_181 1.8', 'module-version Java/1.8.0_181 site_default', - 'module-alias JavaAlias Java/1.8.0_181', + 'module-alias Java/Alias toy/42.1337', + # 'module-alias Java/NonExist non_existant/1', # (only) LMod has this in module avail, disable for now + 'module-alias JavaAlias Java/1.8.0_181', # LMod 7+ only ]) write_file(os.path.join(java_mod_dir, '.modulerc'), modulerc_tcl_txt) @@ -309,13 +309,18 @@ def test_exist(self): self.assertTrue('Java/1.8' in avail_mods) self.assertTrue('Java/site_default' in avail_mods) self.assertTrue('JavaAlias' in avail_mods) + self.assertEqual(self.modtool.exist(['JavaAlias']), [True]) self.assertEqual(self.modtool.exist(['Java/1.8', 'Java/1.8.0_181']), [True, True]) - # check for an alias with a different version suffix than the base module + # module-version with different version suffix than the base module self.assertEqual(self.modtool.exist(['Java/site_default']), [True]) - # And completely different name - self.assertEqual(self.modtool.exist(['JavaAlias']), [True]) + # Check for aliases: + # - completely different nameTrue, True, + # - alias to non existant module + # Skipped for EnvironmentModulesC as module-alias not working correctly there + if self.modtool.__class__ != EnvironmentModulesC: + self.assertEqual(self.modtool.exist(['Java/Alias', 'Java/NonExist']), [True, False]) reset_module_caches() From 1454dd34da9bfc252f8c77337fcd3efc1892b1f7 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 29 Apr 2020 17:00:43 +0200 Subject: [PATCH 13/16] Remove broken test for #368 The issue was caused by self.show returning stdout and stderr As it now returns only stderr the issue is no longer possible --- test/framework/modules.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 0c269748ae..2434e837f8 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -722,24 +722,6 @@ def test_modulefile_path(self): res = modtool.modulefile_path('bzip2/.1.0.6', strip_ext=True) self.assertTrue(res.endswith('test/framework/modules/bzip2/.1.0.6')) - # hack into 'module show GCC/6.4.0-2.28' cache and inject alternate output that modulecmd.tcl sometimes produces - # make sure we only extract the module file path, nothing else... - # cfr. https://github.com/easybuilders/easybuild/issues/368 - modulepath = os.environ['MODULEPATH'].split(':') - mod_show_cache_key = modtool.mk_module_cache_key('GCC/6.4.0-2.28') - mod.MODULE_SHOW_CACHE[mod_show_cache_key] = '\n'.join([ - "import os", - "os.environ['MODULEPATH_modshare'] = '%s'" % ':'.join(m + ':1' for m in modulepath), - "os.environ['MODULEPATH'] = '%s'" % ':'.join(modulepath), - "------------------------------------------------------------------------------", - "%s:" % gcc_mod_file, - "------------------------------------------------------------------------------", - # remainder of output doesn't really matter in this context - "setenv EBROOTGCC /prefix/GCC/6.4.0-2.28" - ]) - res = modtool.modulefile_path('GCC/6.4.0-2.28') - self.assertTrue(os.path.samefile(res, os.path.join(test_dir, 'modules', 'GCC', '6.4.0-2.28'))) - reset_module_caches() def test_path_to_top_of_module_tree(self): From dace69d1d90c7e4f0c0a475cc74733115479cf7e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 30 Apr 2020 16:02:41 +0200 Subject: [PATCH 14/16] Improve module_wrapper_exists deprecation error --- easybuild/tools/modules.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 18ef97ed3d..8fb551704a 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -489,7 +489,8 @@ def module_wrapper_exists(self, mod_name, modulerc_fn='.modulerc', mod_wrapper_r Only .modulerc file in Tcl syntax is considered here. DEPRECATED. Use exists() """ - self.log.deprecated('module_wrapper_exists is unreliable and should no longer be used', '5.0') + self.log.deprecated('module_wrapper_exists is unreliable and should no longer be used. ' + + 'Use exists instead to check for an existing module or alias.', '5.0') if mod_wrapper_regex_template is None: mod_wrapper_regex_template = "^[ ]*module-version (?P[^ ]*) %s$" From 20aab9f68ac58984c69f56b06e67cea0066ab911 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 30 Apr 2020 16:41:29 +0200 Subject: [PATCH 15/16] Move mod_exists_regex_template deprecation notice --- easybuild/tools/modules.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 8fb551704a..9d5f8ce952 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -541,6 +541,9 @@ def exist(self, mod_names, mod_exists_regex_template=None, skip_avail=False, may :param skip_avail: skip checking through 'module avail', only check via 'module show' :param maybe_partial: indicates if the module name may be a partial module name """ + if mod_exists_regex_template is not None: + self.log.deprecated('mod_exists_regex_template is no longer used', '5.0') + def mod_exists_via_show(mod_name): """ Helper function to check whether specified module name exists through 'module show'. @@ -564,9 +567,6 @@ def mod_exists_via_show(mod_name): break return res - if mod_exists_regex_template is not None: - self.log.deprecated('mod_exists_regex_template is no longer used', '5.0') - if skip_avail: avail_mod_names = [] elif len(mod_names) == 1: From 8cb5517117f564544eb7823df2bc856f24fc5ce2 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 30 Apr 2020 16:43:35 +0200 Subject: [PATCH 16/16] Simplify temporary HOME dir creation --- test/framework/modules.py | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 2434e837f8..808eb0df76 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -37,7 +37,6 @@ import stat import sys from distutils.version import StrictVersion -from contextlib import contextmanager from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -58,17 +57,6 @@ TEST_MODULES_COUNT = 81 -@contextmanager -def temporary_home_dir(): - tmpdir = tempfile.mkdtemp() - orig_home = os.environ['HOME'] - os.environ['HOME'] = tmpdir - try: - yield tmpdir - finally: - os.environ['HOME'] = orig_home - shutil.rmtree(tmpdir) - class ModulesTest(EnhancedTestCase): """Test cases for modules.""" @@ -375,14 +363,15 @@ def test_exist(self): self.init_testmods() # Sanity check: Module aliases don't exist yet self.assertEqual(self.modtool.exist(['OpenMPI/99', 'OpenMPIAlias']), [False, False]) - with temporary_home_dir() as home_dir: - reset_module_caches() - write_file(os.path.join(home_dir, '.modulerc'), '\n'.join([ - '#%Module', - 'module-version OpenMPI/2.1.2-GCC-6.4.0-2.28 99', - 'module-alias OpenMPIAlias OpenMPI/2.1.2-GCC-6.4.0-2.28', - ])) - self.assertEqual(self.modtool.exist(['OpenMPI/99', 'OpenMPIAlias']), [True, True]) + # Use a temporary dir, not the users HOME + os.environ['HOME'] = tempfile.mkdtemp() + reset_module_caches() + write_file(os.path.join(os.environ['HOME'], '.modulerc'), '\n'.join([ + '#%Module', + 'module-version OpenMPI/2.1.2-GCC-6.4.0-2.28 99', + 'module-alias OpenMPIAlias OpenMPI/2.1.2-GCC-6.4.0-2.28', + ])) + self.assertEqual(self.modtool.exist(['OpenMPI/99', 'OpenMPIAlias']), [True, True]) def test_load(self): """ test if we load one module it is in the loaded_modules """