diff --git a/.flake8.ini b/.flake8.ini deleted file mode 100644 index 53b6e34f43..0000000000 --- a/.flake8.ini +++ /dev/null @@ -1,9 +0,0 @@ -[flake8] -max-line-length = 120 - -# Hound CI runs with Python 3 (no way around it), -# so we need to specify some Python 2 builtins to avoid that it complains about them -# cfr. https://stackoverflow.com/questions/47427916/how-to-config-hound-ci-to-support-python2-7 -builtins = - basestring, - reduce diff --git a/.hound.yml b/.hound.yml index 2916903576..9449e4f823 100644 --- a/.hound.yml +++ b/.hound.yml @@ -1,4 +1,4 @@ # configuration for houndci, see https://houndci.com/configuration#python flake8: enabled: true - config_file: .flake8.ini + config_file: setup.cfg diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 6054ccc3e8..ccf57d9556 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -3,6 +3,28 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v3.6.1 (May 28th 2018) +---------------------- + +bugfix release +- various enhancements, including: + - add support for enabling fallback in sanity check to consider lib64 equivalent for seemingly missing libraries (#2477) + - add GITHUB_LOWER_SOURCE constant (#2491) + - add 'exts_download_dep_fail' as known easyconfig parameter (#2493) + - add support for passing custom messages on failing sanity check for extensions (#2494) + - add definition for fosscuda toolchain (#2507) +- various bug fixes, including: + - make --inject-checksums always re-order source_urls/sources/patches/checksums (#2487) + - fix git remote url in CONTRIBUTING.md (#2490) + - make flake8 happy in easyblock.py (#2492) + - handle missing permissions for adding labels in --new-pr (#2497) + - restore tweaked $TMPDIR value after loading module (for sanity check) (#2498) + - enhance get_module_path function to auto-detect generic vs software-specific easyblock class names (#2502) + - don't blindly overwrite an existing easyconfig in tweak_one (#2504) + - take account that PlaintextKeyring may be provided via keyrings.alt (#2505) + - prepend location for temporary module file to $MODULEPATH with high priority + mark it as default in load_fake_module method (#2506) + + v3.6.0 (April 26th 2018) ------------------------ diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f9161c3748..e2f9886272 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -212,6 +212,7 @@ def __init__(self, ec, hooks=None): # keep track of initial environment we start in, so we can restore it if needed self.initial_environ = copy.deepcopy(os.environ) + self.tweaked_env_vars = {} # should we keep quiet? self.silent = build_option('silent') @@ -1282,7 +1283,14 @@ def load_module(self, mod_paths=None, purge=True): if self.mod_subdir and self.toolchain.name != DUMMY_TOOLCHAIN_NAME: mods.insert(0, self.toolchain.det_short_module_name()) + # pass initial environment, to use it for resetting the environment before loading the modules self.modules_tool.load(mods, mod_paths=all_mod_paths, purge=purge, init_env=self.initial_environ) + + # handle environment variables that need to be updated after loading modules + for var, val in sorted(self.tweaked_env_vars.items()): + self.log.info("Tweaking $%s: %s", var, val) + env.setvar(var, val) + else: self.log.warning("Not loading module, since self.full_mod_name is not set.") @@ -1297,7 +1305,7 @@ def load_fake_module(self, purge=False): fake_mod_path = self.make_module_step(fake=True) # load fake module - self.modules_tool.prepend_module_path(os.path.join(fake_mod_path, self.mod_subdir)) + self.modules_tool.prepend_module_path(os.path.join(fake_mod_path, self.mod_subdir), priority=10000) self.load_module(purge=purge) return (fake_mod_path, env) @@ -1752,6 +1760,14 @@ def prepare_step(self, start_dir=True): self.toolchain.prepare(self.cfg['onlytcmod'], silent=self.silent, rpath_filter_dirs=self.rpath_filter_dirs, rpath_include_dirs=self.rpath_include_dirs) + # keep track of environment variables that were tweaked and need to be restored after environment got reset + # $TMPDIR may be tweaked for OpenMPI 2.x, which doesn't like long $TMPDIR paths... + for var in ['TMPDIR']: + if os.environ.get(var) != self.initial_environ.get(var): + self.tweaked_env_vars[var] = os.environ.get(var) + self.log.info("Found tweaked value for $%s: %s (was: %s)", + var, self.tweaked_env_vars[var], self.initial_environ[var]) + # handle allowed system dependencies for (name, version) in self.cfg['allow_system_deps']: # root is set to name, not an actual path @@ -1859,7 +1875,7 @@ def extensions_step(self, fetch=False): cls, inst = None, None class_name = encode_class_name(ext['name']) - mod_path = get_module_path(class_name) + mod_path = get_module_path(class_name, generic=False) # try instantiating extension-specific class try: @@ -2143,10 +2159,57 @@ def _sanity_check_step_dry_run(self, custom_paths=None, custom_commands=None, ** else: self.log.debug("Skiping RPATH sanity check") + def _sanity_check_step_extensions(self): + """Sanity check on extensions (if any).""" + failed_exts = [] + for ext in self.ext_instances: + success, fail_msg = None, None + res = ext.sanity_check_step() + # if result is a tuple, we expect a (, ) format + if isinstance(res, tuple): + if len(res) != 2: + raise EasyBuildError("Wrong sanity check result type for '%s' extension: %s", ext.name, res) + success, fail_msg = res + else: + # if result of extension sanity check is not a 2-tuple, treat it as a boolean indicating success + success, fail_msg = res, "(see log for details)" + + if not success: + fail_msg = "failing sanity check for '%s' extension: %s" % (ext.name, fail_msg) + failed_exts.append((ext.name, fail_msg)) + self.log.warning(fail_msg) + else: + self.log.info("Sanity check for '%s' extension passed!", ext.name) + + if failed_exts: + overall_fail_msg = "extensions sanity check failed for %d extensions: " % len(failed_exts) + self.log.warning(overall_fail_msg) + self.sanity_check_fail_msgs.append(overall_fail_msg + ', '.join(x[0] for x in failed_exts)) + self.sanity_check_fail_msgs.extend(x[1] for x in failed_exts) + def _sanity_check_step(self, custom_paths=None, custom_commands=None, extension=False): """Real version of sanity_check_step method.""" paths, path_keys_and_check, commands = self._sanity_check_step_common(custom_paths, custom_commands) + # helper function to sanity check (alternatives for) one particular path + def check_path(xs, typ, check_fn): + """Sanity check for one particular path.""" + found = False + for name in xs: + path = os.path.join(self.installdir, name) + if check_fn(path): + self.log.debug("Sanity check: found %s %s in %s" % (typ, name, self.installdir)) + found = True + break + else: + self.log.debug("Could not find %s %s in %s" % (typ, name, self.installdir)) + + return found + + def xs2str(xs): + """Human-readable version of alternative locations for a particular file/directory.""" + return ' or '.join("'%s'" % x for x in xs) + # check sanity check paths for key, (typ, check_fn) in path_keys_and_check.items(): @@ -2156,21 +2219,28 @@ def _sanity_check_step(self, custom_paths=None, custom_commands=None, extension= elif not isinstance(xs, tuple): raise EasyBuildError("Unsupported type '%s' encountered in %s, not a string or tuple", key, type(xs)) - found = False - for name in xs: - path = os.path.join(self.installdir, name) - if check_fn(path): - self.log.debug("Sanity check: found %s %s in %s" % (typ, name, self.installdir)) - found = True - break - else: - self.log.debug("Could not find %s %s in %s" % (typ, name, self.installdir)) + + found = check_path(xs, typ, check_fn) + + # for library files in lib/, also consider fallback to lib64/ equivalent (and vice versa) + if not found and build_option('lib64_fallback_sanity_check'): + xs_alt = None + if all(x.startswith('lib/') for x in xs): + xs_alt = [os.path.join('lib64', *os.path.split(x)[1:]) for x in xs] + elif all(x.startswith('lib64/') for x in xs): + xs_alt = [os.path.join('lib', *os.path.split(x)[1:]) for x in xs] + + if xs_alt: + self.log.info("%s not found at %s in %s, consider fallback locations: %s", + typ, xs2str(xs), self.installdir, xs2str(xs_alt)) + found = check_path(xs_alt, typ, check_fn) + if not found: - self.sanity_check_fail_msgs.append("no %s of %s in %s" % (typ, xs, self.installdir)) - self.log.warning("Sanity check: %s" % self.sanity_check_fail_msgs[-1]) + sanity_check_fail_msg = "no %s found at %s in %s" % (typ, xs2str(xs), self.installdir) + self.sanity_check_fail_msgs.append(sanity_check_fail_msg) + self.log.warning("Sanity check: %s", sanity_check_fail_msg) - cand_paths = ' or '.join(["'%s'" % x for x in xs]) - trace_msg("%s %s found: %s" % (typ, cand_paths, ('FAILED', 'OK')[found])) + trace_msg("%s %s found: %s" % (typ, xs2str(xs), ('FAILED', 'OK')[found])) fake_mod_data = None # only load fake module for non-extensions, and not during dry run @@ -2189,7 +2259,6 @@ def _sanity_check_step(self, custom_paths=None, custom_commands=None, extension= # run sanity check commands for command in commands: - out, ec = run_cmd(command, simple=False, log_ok=False, log_all=False, trace=False) if ec != 0: fail_msg = "sanity check command %s exited with code %s (output: %s)" % (command, ec, out) @@ -2200,12 +2269,9 @@ def _sanity_check_step(self, custom_paths=None, custom_commands=None, extension= trace_msg("running command '%s': %s" % (command, ('FAILED', 'OK')[ec == 0])) + # also run sanity check for extensions (unless we are an extension ourselves) if not extension: - failed_exts = [ext.name for ext in self.ext_instances if not ext.sanity_check_step()] - - if failed_exts: - self.sanity_check_fail_msgs.append("sanity checks for %s extensions failed!" % failed_exts) - self.log.warning("Sanity check: %s" % self.sanity_check_fail_msgs[-1]) + self._sanity_check_step_extensions() # cleanup if fake_mod_data: @@ -2221,21 +2287,21 @@ def _sanity_check_step(self, custom_paths=None, custom_commands=None, extension= # pass or fail if self.sanity_check_fail_msgs: - raise EasyBuildError("Sanity check failed: %s", ', '.join(self.sanity_check_fail_msgs)) + raise EasyBuildError("Sanity check failed: %s", '\n'.join(self.sanity_check_fail_msgs)) else: self.log.debug("Sanity check passed!") - def _set_module_as_default(self): + def _set_module_as_default(self, fake=False): """ - Defining default module Version + Sets the default module version except if we are in dry run - sets the default module version except if we are in dry run. + :param fake: set default for 'fake' module in temporary location """ version = self.full_mod_name.split('/')[-1] if self.dry_run: dry_run_msg("Marked %s v%s as default version" % (self.name, version)) else: - mod_folderpath = os.path.dirname(self.module_generator.get_module_filepath()) + mod_folderpath = os.path.dirname(self.module_generator.get_module_filepath(fake=fake)) self.module_generator.set_as_default(mod_folderpath, version) def cleanup_step(self): @@ -2343,8 +2409,10 @@ def make_module_step(self, fake=False): else: self.log.info("Skipping devel module...") - if build_option('set_default_module'): - self._set_module_as_default() + # always set default for temporary module file, + # to avoid that it gets overruled by an existing module file that is set as default + if fake or build_option('set_default_module'): + self._set_module_as_default(fake=fake) return modpath diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 7f68136361..53c353afe1 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -40,25 +40,25 @@ _log = fancylogger.getLogger('easyconfig.default', fname=False) +# constants for different categories of easyconfig parameters +# use tuples so we can sort them based on the numbers +HIDDEN = (-1, 'hidden') +MANDATORY = (0, 'mandatory') +CUSTOM = (1, 'easyblock-specific') +TOOLCHAIN = (2, 'toolchain') +BUILD = (3, 'build') +FILEMANAGEMENT = (4, 'file-management') +DEPENDENCIES = (5, 'dependencies') +LICENSE = (6, 'license') +EXTENSIONS = (7, 'extensions') +MODULES = (8, 'modules') +OTHER = (9, 'other') + # we use a tuple here so we can sort them based on the numbers -ALL_CATEGORIES = { - 'HIDDEN': (-1, 'hidden'), - 'MANDATORY': (0, 'mandatory'), - 'CUSTOM': (1, 'easyblock-specific'), - 'TOOLCHAIN': (2, 'toolchain'), - 'BUILD': (3, 'build'), - 'FILEMANAGEMENT': (4, 'file-management'), - 'DEPENDENCIES': (5, 'dependencies'), - 'LICENSE': (6, 'license'), - 'EXTENSIONS': (7, 'extensions'), - 'MODULES': (8, 'modules'), - 'OTHER': (9, 'other'), -} -# define constants so they can be used below -# avoid that pylint complains about unknown variables in this file -# pylint: disable=E0602 -globals().update(ALL_CATEGORIES) +CATEGORY_NAMES = ['BUILD', 'CUSTOM', 'DEPENDENCIES', 'EXTENSIONS', 'FILEMANAGEMENT', 'HIDDEN', + 'LICENSE', 'MANDATORY', 'MODULES', 'OTHER', 'TOOLCHAIN'] +ALL_CATEGORIES = dict((name, eval(name)) for name in CATEGORY_NAMES) # List of tuples. Each tuple has the following format (key, [default, help text, category]) DEFAULT_CONFIG = { @@ -151,6 +151,7 @@ 'license_server_port': [None, 'Port for license server', LICENSE], # EXTENSIONS easyconfig parameters + 'exts_download_dep_fail': [False, "Fail if downloaded dependencies are detected for extensions", EXTENSIONS], 'exts_classmap': [{}, "Map of extension name to class for handling build and installation.", EXTENSIONS], 'exts_defaultclass': [None, "List of module for and name of the default extension class", EXTENSIONS], 'exts_default_options': [{}, "List of default options for extensions", EXTENSIONS], @@ -171,8 +172,8 @@ 'moduleclass': ['base', 'Module class to be used for this software', MODULES], 'moduleforceunload': [False, 'Force unload of all modules when loading the extension', MODULES], 'moduleloadnoconflict': [False, "Don't check for conflicts, unload other versions instead ", MODULES], - 'module_depends_on' : [False, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module ' - '(implies recursive unloading of modules).', MODULES], + 'module_depends_on': [False, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module ' + '(implies recursive unloading of modules).', MODULES], 'recursive_module_unload': [False, 'Recursive unload of all dependencies when unloading module', MODULES], # MODULES documentation easyconfig parameters @@ -180,8 +181,8 @@ 'docpaths': [None, "List of paths for documentation relative to installation directory", MODULES], 'examples': [None, "Free-form text with examples on using the software", MODULES], 'site_contacts': [None, "String/list of strings with site contacts for the software", MODULES], - 'upstream_contacts': [None, ("String/list of strings with upstream contact addresses " - "(e.g., support e-mail, mailing list, bugtracker)"), MODULES], + 'upstream_contacts': [None, "String/list of strings with upstream contact addresses " + "(e.g., support e-mail, mailing list, bugtracker)", MODULES], 'usage': [None, "Usage instructions for the software", MODULES], 'whatis': [None, "List of brief (one line) description entries for the software", MODULES], diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 87275d00e5..2a47fcaa09 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -41,12 +41,10 @@ import functools import os import re -import shutil from vsc.utils import fancylogger from vsc.utils.missing import get_class_for, nub from vsc.utils.patterns import Singleton -import easybuild.tools.environment as env from easybuild.framework.easyconfig import MANDATORY from easybuild.framework.easyconfig.constants import EXTERNAL_MODULE_MARKER from easybuild.framework.easyconfig.default import DEFAULT_CONFIG @@ -60,7 +58,8 @@ from easybuild.toolchains.gcccore import GCCcore from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, get_module_naming_scheme -from easybuild.tools.filetools import copy_file, decode_class_name, encode_class_name, mkdir, read_file, write_file +from easybuild.tools.filetools import EASYBLOCK_CLASS_PREFIX +from easybuild.tools.filetools import copy_file, decode_class_name, encode_class_name, read_file, write_file from easybuild.tools.module_naming_scheme import DEVEL_MODULE_SUFFIX from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name @@ -785,7 +784,7 @@ def _parse_dependency(self, dep, hidden=False, build_only=False): dependency.update(dep) # make sure 'dummy' key is handled appropriately - if 'dummy' in dep and not 'toolchain' in dep: + if 'dummy' in dep and 'toolchain' not in dep: dependency['toolchain'] = dep['dummy'] if dep.get('external_module', False): @@ -932,7 +931,6 @@ def generate_template_values(self): # recursive call, until there are no more changes to template values; # important since template values may include other templates - prev_template_values = None cont = True while cont: cont = False @@ -1084,14 +1082,8 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error class_name, modulepath) cls = get_class_for(modulepath, class_name) else: - # if we only get the class name, most likely we're dealing with a generic easyblock - try: - modulepath = get_module_path(easyblock, generic=True) - cls = get_class_for(modulepath, class_name) - except ImportError, err: - # we might be dealing with a non-generic easyblock, e.g. with --easyblock is used - modulepath = get_module_path(easyblock) - cls = get_class_for(modulepath, class_name) + modulepath = get_module_path(easyblock) + cls = get_class_for(modulepath, class_name) _log.info("Derived full easyblock module path for %s: %s" % (class_name, modulepath)) else: # if no easyblock specified, try to find if one exists @@ -1100,7 +1092,7 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error # The following is a generic way to calculate unique class names for any funny software title class_name = encode_class_name(name) # modulepath will be the namespace + encoded modulename (from the classname) - modulepath = get_module_path(class_name) + modulepath = get_module_path(class_name, generic=False) modulepath_imported = False try: __import__(modulepath, globals(), locals(), ['']) @@ -1113,7 +1105,7 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error _log.debug("Module path '%s' found" % modulepath) else: _log.debug("No module path '%s' found" % modulepath) - modulepath_bis = get_module_path(name, decode=False) + modulepath_bis = get_module_path(name, generic=False, decode=False) _log.debug("Module path determined based on software name: %s" % modulepath_bis) if modulepath_bis != modulepath: _log.nosupport("Determining module path based on software name", '2.0') @@ -1151,14 +1143,20 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error raise EasyBuildError("Failed to obtain class for %s easyblock (not available?): %s", easyblock, err) -def get_module_path(name, generic=False, decode=True): +def get_module_path(name, generic=None, decode=True): """ Determine the module path for a given easyblock or software name, based on the encoded class name. + + :param generic: whether or not the easyblock is generic (if None: auto-derive from specified class name) + :param decode: whether or not to decode the provided class name """ if name is None: return None + if generic is None: + generic = not name.startswith(EASYBLOCK_CLASS_PREFIX) + # example: 'EB_VSC_minus_tools' should result in 'vsc_tools' if decode: name = decode_class_name(name) @@ -1205,8 +1203,7 @@ def resolve_template(value, tmpl_dict): try: value = value % tmpl_dict except KeyError: - _log.warning("Unable to resolve template value %s with dict %s" % - (value, tmpl_dict)) + _log.warning("Unable to resolve template value %s with dict %s", value, tmpl_dict) else: # this block deals with references to objects and returns other references # for reading this is ok, but for self['x'] = {} diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 7c50715112..ed3fc0362f 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -44,7 +44,8 @@ from easybuild.framework.easyconfig.default import get_easyconfig_parameter_default from easybuild.framework.easyconfig.easyconfig import EasyConfig, create_paths, process_easyconfig -from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.build_log import EasyBuildError, print_warning +from easybuild.tools.config import build_option from easybuild.tools.filetools import read_file, write_file from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.robot import resolve_dependencies @@ -108,7 +109,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): # generate tweaked easyconfigs, and continue with those instead tweaked_easyconfigs = [] - for orig_ec in orig_ecs: + for orig_ec in orig_ecs: # Only return tweaked easyconfigs for easyconfigs which were listed originally on the command line (and use the # prepended path so that they are found first). # easyconfig files for dependencies are also generated but not included, they will be resolved via --robot @@ -124,7 +125,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): return tweaked_easyconfigs -def tweak_one(src_fn, target_fn, tweaks, targetdir=None): +def tweak_one(orig_ec, tweaked_ec, tweaks, targetdir=None): """ Tweak an easyconfig file with the given list of tweaks, using replacement via regular expressions. Note: this will only work 'well-written' easyconfig files, i.e. ones that e.g. set the version @@ -134,14 +135,21 @@ def tweak_one(src_fn, target_fn, tweaks, targetdir=None): The tweaks should be specified in a dictionary, with parameters and keys that map to the values to be set. - Reads easyconfig file at path , and writes the tweaked easyconfig file to . + Reads easyconfig file at path , and writes the tweaked easyconfig file to . - If no target filename is provided, a target filepath is generated based on the contents of - the tweaked easyconfig file. + If is not provided, a target filepath is generated based on and the + contents of the tweaked easyconfig file. + + :param orig_ec: location of original easyconfig file to read + :param tweaked_ec: location where tweaked easyconfig file should be written + (if this is None, then filename for tweaked easyconfig is auto-derived from contents) + :param tweaks: dictionary with set of changes to apply to original easyconfig file + :param targetdir: target directory for tweaked easyconfig file, defaults to temporary directory + (only used if tweaked_ec is None) """ # read easyconfig file - ectxt = read_file(src_fn) + ectxt = read_file(orig_ec) _log.debug("Contents of original easyconfig file, prior to tweaking:\n%s" % ectxt) # determine new toolchain if it's being changed @@ -151,7 +159,7 @@ def tweak_one(src_fn, target_fn, tweaks, targetdir=None): tc_regexp = re.compile(r"^\s*toolchain\s*=\s*(.*)$", re.M) res = tc_regexp.search(ectxt) if not res: - raise EasyBuildError("No toolchain found in easyconfig file %s: %s", src_fn, ectxt) + raise EasyBuildError("No toolchain found in easyconfig file %s: %s", orig_ec, ectxt) toolchain = eval(res.group(1)) for key in ['name', 'version']: @@ -170,7 +178,7 @@ def __repr__(self): additions = [] - # automagically clear out list of checksums if software version is being tweaked + # automagically clear out list of checksums if software version is being tweaked if 'version' in tweaks and 'checksums' not in tweaks: tweaks['checksums'] = [] _log.warning("Tweaking version: checksums cleared, verification disabled.") @@ -239,7 +247,7 @@ def __repr__(self): _log.debug("Contents of tweaked easyconfig file:\n%s" % ectxt) # come up with suiting file name for tweaked easyconfig file if none was specified - if target_fn is None: + if tweaked_ec is None: fn = None try: # obtain temporary filename @@ -259,14 +267,21 @@ def __repr__(self): if targetdir is None: targetdir = tempfile.gettempdir() - target_fn = os.path.join(targetdir, fn) - _log.debug("Generated file name for tweaked easyconfig file: %s" % target_fn) + tweaked_ec = os.path.join(targetdir, fn) + _log.debug("Generated file name for tweaked easyconfig file: %s", tweaked_ec) # write out tweaked easyconfig file - write_file(target_fn, ectxt) - _log.info("Tweaked easyconfig file written to %s" % target_fn) + if os.path.exists(tweaked_ec): + if build_option('force'): + print_warning("Overwriting existing file at %s with tweaked easyconfig file (due to --force)", tweaked_ec) + else: + raise EasyBuildError("A file already exists at %s where tweaked easyconfig file would be written", + tweaked_ec) + + write_file(tweaked_ec, ectxt) + _log.info("Tweaked easyconfig file written to %s", tweaked_ec) - return target_fn + return tweaked_ec def pick_version(req_ver, avail_vers): diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index c6ff2c0b8e..dacde92953 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -133,6 +133,7 @@ def sanity_check_step(self): """ Sanity check to run after installing extension """ + res = (True, '') if os.path.isdir(self.installdir): change_dir(self.installdir) @@ -146,7 +147,7 @@ def sanity_check_step(self): cmd, inp = exts_filter else: self.log.debug("no exts_filter setting found, skipping sanitycheck") - return True + cmd = None if 'modulename' in self.options: modname = self.options['modulename'] @@ -155,10 +156,10 @@ def sanity_check_step(self): modname = self.name self.log.debug("self.name: %s", modname) - if modname == False: - # allow skipping of sanity check by setting module name to False - return True - else: + # allow skipping of sanity check by setting module name to False + if modname is False: + self.log.info("modulename set to False for '%s' extension, so skipping sanity check", self.name) + elif cmd: template = { 'ext_name': modname, 'ext_version': self.version, @@ -178,9 +179,15 @@ def sanity_check_step(self): (output, ec) = run_cmd(cmd, log_ok=False, simple=False, regexp=False, inp=stdin) if ec: - msg = "%s failed to install, cmd '%s' (stdin: %s) output: %s" % (self.name, cmd, stdin, output) - self.log.warn("Extension: %s" % msg) - self.sanity_check_fail_msgs.append(msg) - return False - else: - return True + if stdin: + fail_msg = 'command "%s" (stdin: "%s") failed' % (cmd, stdin) + else: + fail_msg = 'command "%s" failed' % cmd + fail_msg += "; output:\n%s" % output.strip() + self.log.warning("Sanity check for '%s' extension failed: %s", self.name, fail_msg) + res = (False, fail_msg) + # keep track of all reasons of failure + # (only relevant when this extension is installed stand-alone via ExtensionEasyBlock) + self.sanity_check_fail_msgs.append(fail_msg) + + return res diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index 26952cc175..a4e6af0f1f 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -127,8 +127,8 @@ def sanity_check_step(self, exts_filter=None, custom_paths=None, custom_commands # load fake module fake_mod_data = self.load_fake_module(purge=True) - # perform sanity check - sanity_check_ok = Extension.sanity_check_step(self) + # perform extension sanity check + (sanity_check_ok, fail_msg) = Extension.sanity_check_step(self) if fake_mod_data: # unload fake module and clean up @@ -140,21 +140,19 @@ def sanity_check_step(self, exts_filter=None, custom_paths=None, custom_commands extension=self.is_extension) # pass or fail sanity check - if not sanity_check_ok: - msg = "Sanity check for %s failed: %s" % (self.name, '; '.join(self.sanity_check_fail_msgs)) - if self.is_extension: - self.log.warning(msg) - else: - raise EasyBuildError(msg) - return False + if sanity_check_ok: + self.log.info("Sanity check for %s successful!", self.name) else: - self.log.info("Sanity check for %s successful!" % self.name) - return True + if not self.is_extension: + msg = "Sanity check for %s failed: %s" % (self.name, '; '.join(self.sanity_check_fail_msgs)) + raise EasyBuildError(msg) + + return (sanity_check_ok, '; '.join(self.sanity_check_fail_msgs)) def make_module_extra(self, extra=None): """Add custom entries to module.""" txt = EasyBlock.make_module_extra(self) - if not extra is None: + if extra is not None: txt += extra return txt diff --git a/easybuild/main.py b/easybuild/main.py index 9141ebea00..0ee5958530 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -389,10 +389,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if try_to_generate and build_specs and not generated_ecs: easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths) - # create a container if options.containerize: - _log.info("Creating %s container" % options.container_type) - containerize(easyconfigs, options.container_type) + # if --containerize/-C create a container recipe (and optionally container image), and stop + containerize(easyconfigs) cleanup(logfile, eb_tmpdir, testing) sys.exit(0) diff --git a/easybuild/toolchains/fosscuda.py b/easybuild/toolchains/fosscuda.py new file mode 100644 index 0000000000..baa5b3e9f0 --- /dev/null +++ b/easybuild/toolchains/fosscuda.py @@ -0,0 +1,40 @@ +## +# Copyright 2013-2018 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for fosscuda compiler toolchain (includes GCC+CUDA, OpenMPI, OpenBLAS, LAPACK, ScaLAPACK and FFTW). + +:author: Davide Vanzo (Vanderbilt University) +""" + +from easybuild.toolchains.gompic import Gompic +from easybuild.toolchains.fft.fftw import Fftw +from easybuild.toolchains.linalg.openblas import OpenBLAS +from easybuild.toolchains.linalg.scalapack import ScaLAPACK + + +class Fosscuda(Gompic, OpenBLAS, ScaLAPACK, Fftw): + """Compiler toolchain with GCC+CUDA, OpenMPI, OpenBLAS, ScaLAPACK and FFTW.""" + NAME = 'fosscuda' + SUBTOOLCHAIN = Gompic.NAME diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index e26ca76f0d..f3d3952b1d 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -189,6 +189,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'hidden', 'ignore_checksums', 'install_latest_eb_release', + 'lib64_fallback_sanity_check', 'logtostdout', 'minimal_toolchains', 'module_only', diff --git a/easybuild/tools/containers/__init__.py b/easybuild/tools/containers/__init__.py index e444d35e44..f81af626f9 100644 --- a/easybuild/tools/containers/__init__.py +++ b/easybuild/tools/containers/__init__.py @@ -31,7 +31,7 @@ """ from vsc.utils import fancylogger -from easybuild.tools.config import CONT_TYPE_SINGULARITY, CONT_TYPE_DOCKER +from easybuild.tools.config import CONT_TYPE_SINGULARITY, CONT_TYPE_DOCKER, build_option from easybuild.tools.build_log import EasyBuildError from .singularity import singularity as singularity_containerize from .docker import docker_containerize @@ -39,12 +39,15 @@ _log = fancylogger.getLogger('tools.containers') # pylint: disable=C0103 -def containerize(easyconfigs, container_type): +def containerize(easyconfigs): """ Generate container recipe + (optionally) image """ _log.experimental("support for generating container recipes and images (--containerize/-C)") + container_type = build_option('container_type') + _log.info("Creating %s container", container_type) + if container_type == CONT_TYPE_SINGULARITY: singularity_containerize(easyconfigs) elif container_type == CONT_TYPE_DOCKER: diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 029c0446a7..33828b63a3 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1148,11 +1148,12 @@ def new_pr(paths, ecs, title=None, descr=None, commit_msg=None): # post labels pr = data['html_url'].split('/')[-1] pr_url = g.repos[pr_target_account][pr_target_repo].issues[pr] - status, data = pr_url.labels.post(body=labels) - if not status == HTTP_STATUS_OK: - raise EasyBuildError("Failed to add labels to PR# %s; status %s, data: %s", pr, status, data) - - print_msg("Added labels %s to PR#%s" % (', '.join(labels), pr), log=_log, prefix=False) + try: + status, data = pr_url.labels.post(body=labels) + if status == HTTP_STATUS_OK: + print_msg("Added labels %s to PR#%s" % (', '.join(labels), pr), log=_log, prefix=False) + except urllib2.HTTPError as err: + _log.info("Failed to add labels to PR# %s: %s." % (pr, err)) @only_if_module_is_available('git', pkgname='GitPython') diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index e4e16f1cd5..7d9a4d9dc9 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -45,7 +45,7 @@ from vsc.utils.missing import get_subclasses from easybuild.tools.build_log import EasyBuildError, print_warning -from easybuild.tools.config import ERROR, IGNORE, PURGE, UNLOAD, UNSET, WARN +from easybuild.tools.config import ERROR, IGNORE, PURGE, UNLOAD, UNSET from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, LOADED_MODULES_ACTIONS from easybuild.tools.config import build_option, get_modules_tool, install_path from easybuild.tools.environment import ORIG_OS_ENVIRON, restore_env, setvar, unset_env_vars @@ -203,11 +203,6 @@ def buildstats(self): """Return tuple with data to be included in buildstats""" return (self.__class__.__name__, self.cmd, self.version) - @property - def modules(self): - """(NO LONGER SUPPORTED!) Property providing access to 'modules' class variable""" - self.log.nosupport("'modules' class variable is not supported anymore, use load([]) instead", '2.0') - def set_and_check_version(self): """Get the module version, and check any requirements""" if self.COMMAND in MODULE_VERSION_CACHE: @@ -330,8 +325,17 @@ def set_mod_paths(self, mod_paths=None): self.log.debug("$MODULEPATH after set_mod_paths: %s" % os.environ.get('MODULEPATH', '')) - def use(self, path): - """Add module path via 'module use'.""" + def use(self, path, priority=None): + """ + Add path to $MODULEPATH via 'module use'. + + :param path: path to add to $MODULEPATH + :param priority: priority for this path in $MODULEPATH (Lmod-specific) + """ + if priority: + self.log.info("Ignoring specified priority '%s' when running 'module use %s' (Lmod-specific)", + priority, path) + # make sure path exists before we add it mkdir(path, parents=True) self.run_module(['use', path]) @@ -367,13 +371,18 @@ def remove_module_path(self, path, set_mod_paths=True): if set_mod_paths: self.set_mod_paths() - def prepend_module_path(self, path, set_mod_paths=True): + def prepend_module_path(self, path, set_mod_paths=True, priority=None): """ Prepend given module path to list of module paths, or bump it to 1st place. :param path: path to prepend to $MODULEPATH :param set_mod_paths: (re)set self.mod_paths + :param priority: priority for this path in $MODULEPATH (Lmod-specific) """ + if priority: + self.log.info("Ignoring specified priority '%s' when prepending %s to $MODULEPATH (Lmod-specific)", + priority, path) + # generic approach: remove the path first (if it's there), then add it again (to the front) modulepath = curr_module_paths() if not modulepath: @@ -489,10 +498,6 @@ def mod_exists_via_show(mod_name): return mods_exist - def exists(self, mod_name): - """NO LONGER SUPPORTED: use exist method instead""" - self.log.nosupport("exists() is not supported anymore, use exist([]) instead", '2.0') - def load(self, modules, mod_paths=None, purge=False, init_env=None, allow_reload=True): """ Load all requested modules. @@ -532,9 +537,6 @@ def unload(self, modules=None): """ Unload all requested modules. """ - if modules is None: - self.log.nosupport("Unloading modules listed in _modules class variable", '2.0') - for mod in modules: self.run_module('unload', mod) @@ -643,14 +645,6 @@ def run_module(self, *args, **kwargs): else: args = list(args) - module_path_key = None - if 'mod_paths' in kwargs: - module_path_key = 'mod_paths' - elif 'modulePath' in kwargs: - module_path_key = 'modulePath' - if module_path_key is not None: - self.log.nosupport("Use of '%s' named argument in 'run_module'" % module_path_key, '2.0') - self.log.debug('Current MODULEPATH: %s' % os.environ.get('MODULEPATH', '')) # restore selected original environment variables before running module command @@ -854,7 +848,10 @@ def interpret_raw_path_tcl(self, txt): res = txt.strip('"') # first interpret (outer) 'file join' statement (if any) - file_join = lambda res: os.path.join(*[x.strip('"') for x in res.groups()]) + def file_join(res): + """Helper function to compose joined path.""" + return os.path.join(*[x.strip('"') for x in res.groups()]) + res = re.sub('\[\s+file\s+join\s+(.*)\s+(.*)\s+\]', file_join, res) # also interpret all $env(...) parts @@ -1017,6 +1014,7 @@ def update(self): """Update after new modules were added.""" pass + class EnvironmentModulesTcl(EnvironmentModulesC): """Interface to (Tcl) environment modules (modulecmd.tcl).""" # Tcl environment modules have no --terse (yet), @@ -1129,7 +1127,7 @@ def __init__(self, *args, **kwargs): def check_module_function(self, *args, **kwargs): """Check whether selected module tool matches 'module' function definition.""" - if not 'regex' in kwargs: + if 'regex' not in kwargs: kwargs['regex'] = r".*(%s|%s)" % (self.COMMAND, self.COMMAND_ENVIRONMENT) super(Lmod, self).check_module_function(*args, **kwargs) @@ -1212,17 +1210,33 @@ def update(self): except (IOError, OSError), err: raise EasyBuildError("Failed to update Lmod spider cache %s: %s", cache_fp, err) - def prepend_module_path(self, path, set_mod_paths=True): + def use(self, path, priority=None): + """ + Add path to $MODULEPATH via 'module use'. + + :param path: path to add to $MODULEPATH + :param priority: priority for this path in $MODULEPATH (Lmod-specific) + """ + # make sure path exists before we add it + mkdir(path, parents=True) + + if priority: + self.run_module(['use', '--priority', str(priority), path]) + else: + self.run_module(['use', path]) + + def prepend_module_path(self, path, set_mod_paths=True, priority=None): """ Prepend given module path to list of module paths, or bump it to 1st place. :param path: path to prepend to $MODULEPATH :param set_mod_paths: (re)set self.mod_paths + :param priority: priority for this path in $MODULEPATH (Lmod-specific) """ # Lmod pushes a path to the front on 'module use', no need for (costly) 'module unuse' modulepath = curr_module_paths() if not modulepath or os.path.realpath(modulepath[0]) != os.path.realpath(path): - self.use(path) + self.use(path, priority=priority) if set_mod_paths: self.set_mod_paths() @@ -1251,15 +1265,11 @@ def get_software_root(name, with_env_var=False): Return the software root set for a particular software name. """ env_var = get_software_root_env_var_name(name) - legacy_key = "SOFTROOT%s" % convert_name(name, upper=True) root = None if env_var in os.environ: root = os.getenv(env_var) - elif legacy_key in os.environ: - _log.nosupport("Legacy env var %s is being relied on!" % legacy_key, "2.0") - if with_env_var: res = (root, env_var) else: @@ -1317,16 +1327,14 @@ def get_software_version(name): Return the software version set for a particular software name. """ env_var = get_software_version_env_var_name(name) - legacy_key = "SOFTVERSION%s" % convert_name(name, upper=True) version = None if env_var in os.environ: version = os.getenv(env_var) - elif legacy_key in os.environ: - _log.nosupport("Legacy env var %s is being relied on!" % legacy_key, "2.0") return version + def curr_module_paths(): """ Return a list of current module paths. diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 77427e50f7..339bf1f33d 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -322,7 +322,8 @@ def override_options(self): descr = ("Override options", "Override default EasyBuild behavior.") opts = OrderedDict({ - 'add-dummy-to-minimal-toolchains': ("Include dummy in minimal toolchain searches", None, 'store_true', False), + 'add-dummy-to-minimal-toolchains': ("Include dummy in minimal toolchain searches", + None, 'store_true', False), 'allow-loaded-modules': ("List of software names for which to allow loaded modules in initial environment", 'strlist', 'store', DEFAULT_ALLOW_LOADED_MODULES), 'allow-modules-tool-mismatch': ("Allow mismatch of modules tool and definition of 'module' function", @@ -373,7 +374,8 @@ def override_options(self): 'group': ("Group to be used for software installations (only verified, not set)", None, 'store', None), 'group-writable-installdir': ("Enable group write permissions on installation directory after installation", None, 'store_true', False), - 'hidden': ("Install 'hidden' module file(s) by prefixing their version with '.'", None, 'store_true', False), + 'hidden': ("Install 'hidden' module file(s) by prefixing their version with '.'", + None, 'store_true', False), 'hide-deps': ("Comma separated list of dependencies that you want automatically hidden, " "(e.g. --hide-deps=zlib,ncurses)", 'strlist', 'extend', None), 'hide-toolchains': ("Comma separated list of toolchains that you want automatically hidden, " @@ -381,6 +383,8 @@ def override_options(self): 'ignore-checksums': ("Ignore failing checksum verification", None, 'store_true', False), 'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False), 'install-latest-eb-release': ("Install latest known version of easybuild", None, 'store_true', False), + 'lib64-fallback-sanity-check': ("Fallback in sanity check to lib64/ equivalent for missing libraries", + None, 'store_true', True), 'max-fail-ratio-adjust-permissions': ("Maximum ratio for failures to allow when adjusting permissions", 'float', 'store', DEFAULT_MAX_FAIL_RATIO_PERMS), 'minimal-toolchains': ("Use minimal toolchain when resolving dependencies", None, 'store_true', False), diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 94311e56ab..60d7e2ee75 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('3.6.1.dev0') +VERSION = LooseVersion('3.6.2.dev0') UNKNOWN = 'UNKNOWN' diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index b7661f8e67..a20a9d98dc 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -44,8 +44,7 @@ from easybuild.tools import config from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import get_module_syntax -from easybuild.tools.filetools import copy_file, mkdir, read_file, remove_file, write_file -from easybuild.tools.modules import modules_tool +from easybuild.tools.filetools import copy_dir, copy_file, mkdir, read_file, remove_file, write_file from easybuild.tools.version import get_git_revision, this_is_easybuild @@ -147,6 +146,64 @@ def extra_options(): eb.close_log() os.remove(eb.logfile) + def test_load_module(self): + """Test load_module method.""" + # copy OpenMPI module used in gompi/1.3.12 to fiddle with it, i.e. to fake bump OpenMPI version used in it + tmp_modules = os.path.join(self.test_prefix, 'modules') + mkdir(tmp_modules) + + test_dir = os.path.abspath(os.path.dirname(__file__)) + copy_dir(os.path.join(test_dir, 'modules', 'OpenMPI'), os.path.join(tmp_modules, 'OpenMPI')) + + openmpi_module = os.path.join(tmp_modules, 'OpenMPI', '1.6.4-GCC-4.6.4') + ompi_mod_txt = read_file(openmpi_module) + write_file(openmpi_module, ompi_mod_txt.replace('1.6.4', '2.0.2')) + + self.modtool.use(tmp_modules) + + orig_tmpdir = os.path.join(self.test_prefix, 'verylongdirectorythatmaycauseproblemswithopenmpi2') + os.environ['TMPDIR'] = orig_tmpdir + + self.contents = '\n'.join([ + "easyblock = 'ConfigureMake'", + "name = 'pi'", + "version = '3.14'", + "homepage = 'http://example.com'", + "description = 'test easyconfig'", + "toolchain = {'name': 'gompi', 'version': '1.3.12'}", + ]) + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + eb.installdir = config.build_path() + + # $TMPDIR is not touched yet at this point + self.assertEqual(os.environ.get('TMPDIR'), orig_tmpdir) + + self.mock_stderr(True) + self.mock_stdout(True) + eb.prepare_step(start_dir=False) + stderr = self.get_stderr() + stdout = self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertFalse(stdout) + self.assertTrue(stderr.strip().startswith("WARNING: Long $TMPDIR path may cause problems with OpenMPI 2.x")) + + # we expect $TMPDIR to be tweaked by the prepare step (OpenMPI 2.x doesn't like long $TMPDIR values) + tweaked_tmpdir = os.environ.get('TMPDIR') + self.assertTrue(tweaked_tmpdir != orig_tmpdir) + + eb.make_module_step() + eb.load_module() + + # $TMPDIR does *not* get reset to original value after loading of module + # (which involves resetting the environment before loading the module) + self.assertEqual(os.environ.get('TMPDIR'), tweaked_tmpdir) + + # cleanup + eb.close_log() + os.remove(eb.logfile) + def test_fake_module_load(self): """Testcase for fake module load""" self.contents = '\n'.join([ @@ -161,6 +218,21 @@ def test_fake_module_load(self): eb = EasyBlock(EasyConfig(self.eb_file)) eb.installdir = config.build_path() fake_mod_data = eb.load_fake_module() + + pi_modfile = os.path.join(fake_mod_data[0], 'pi', '3.14') + if get_module_syntax() == 'Lua': + pi_modfile += '.lua' + + self.assertTrue(os.path.exists(pi_modfile)) + + # check whether temporary module file is marked as default + if get_module_syntax() == 'Lua': + default_symlink = os.path.join(fake_mod_data[0], 'pi', 'default') + self.assertTrue(os.path.samefile(default_symlink, pi_modfile)) + else: + dot_version_txt = read_file(os.path.join(fake_mod_data[0], 'pi', '.version')) + self.assertTrue("set ModulesVersion 3.14" in dot_version_txt) + eb.clean_up_fake_module(fake_mod_data) # cleanup @@ -1090,15 +1162,31 @@ def test_patch_step(self): def test_extensions_sanity_check(self): """Test sanity check aspect of extensions.""" test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-1.3.12-test.eb')) + toy_ec_fn = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-1.3.12-test.eb') + + # this import only works here, since EB_toy is a test easyblock + from easybuild.easyblocks.toy import EB_toy + + # purposely inject failing custom extension filter for last extension + toy_ec = EasyConfig(toy_ec_fn) + toy_ec.enable_templating = False + exts_list = toy_ec['exts_list'] + exts_list[-1][2]['exts_filter'] = ("thisshouldfail", '') + toy_ec['exts_list'] = exts_list + toy_ec.enable_templating = True + + eb = EB_toy(toy_ec) + eb.silent = True + error_pattern = r"Sanity check failed: extensions sanity check failed for 1 extensions: toy\n" + error_pattern += r"failing sanity check for 'toy' extension: " + error_pattern += r'command "thisshouldfail" failed; output:\n/bin/bash: thisshouldfail: command not found' + self.assertErrorRegex(EasyBuildError, error_pattern, eb.run_all_steps, True) # purposely put sanity check command in place that breaks the build, # to check whether sanity check is only run once; # sanity check commands are checked after checking sanity check paths, so this should work + toy_ec = EasyConfig(toy_ec_fn) toy_ec.update('sanity_check_commands', [("%(installdir)s/bin/toy && rm %(installdir)s/bin/toy", '')]) - - # this import only works here, since EB_toy is a test easyblock - from easybuild.easyblocks.toy import EB_toy eb = EB_toy(toy_ec) eb.silent = True eb.run_all_steps(True) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 50ffdd4723..c89bfe4983 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -47,8 +47,8 @@ from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig.constants import EXTERNAL_MODULE_MARKER from easybuild.framework.easyconfig.easyconfig import ActiveMNS, EasyConfig, create_paths, copy_easyconfigs -from easybuild.framework.easyconfig.easyconfig import letter_dir_for, get_easyblock_class, process_easyconfig -from easybuild.framework.easyconfig.easyconfig import resolve_template, verify_easyconfig_filename +from easybuild.framework.easyconfig.easyconfig import get_easyblock_class, get_module_path, letter_dir_for +from easybuild.framework.easyconfig.easyconfig import process_easyconfig, resolve_template, verify_easyconfig_filename from easybuild.framework.easyconfig.licenses import License, LicenseGPLv3 from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig from easybuild.framework.easyconfig.templates import template_constant_dict, to_template_str @@ -59,7 +59,8 @@ from easybuild.tools.config import module_classes from easybuild.tools.configobj import ConfigObj from easybuild.tools.docs import avail_easyconfig_constants, avail_easyconfig_templates -from easybuild.tools.filetools import adjust_permissions, copy_file, mkdir, read_file, symlink, which, write_file +from easybuild.tools.filetools import adjust_permissions, copy_file, mkdir, read_file, remove_file, symlink +from easybuild.tools.filetools import which, write_file from easybuild.tools.module_naming_scheme.toolchain import det_toolchain_compilers, det_toolchain_mpi from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.options import parse_external_modules_metadata @@ -380,6 +381,8 @@ def test_tweaking(self): fd, tweaked_fn = tempfile.mkstemp(prefix='easybuild-tweaked-', suffix='.eb') os.close(fd) + remove_file(tweaked_fn) + patches = ["t1.patch", ("t2.patch", 1), ("t3.patch", "test"), ("t4.h", "include")] self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', @@ -416,6 +419,8 @@ def test_tweaking(self): self.assertEqual(eb['toolchain']['version'], tcver) self.assertEqual(eb['patches'], new_patches) + remove_file(tweaked_fn) + eb = EasyConfig(self.eb_file) # eb['toolchain']['version'] = tcver does not work as expected with templating enabled eb.enable_templating = False @@ -559,6 +564,7 @@ def test_obtain_easyconfig(self): res = obtain_ec_for(specs, [self.ec_dir], None) self.assertEqual(res[0], False) self.assertEqual(res[1], os.path.join(self.ec_dir, fns[-1])) + remove_file(res[1]) # should not pick between multiple available toolchain names name = "pi" @@ -590,7 +596,7 @@ def test_obtain_easyconfig(self): self.assertEqual(ec['versionsuffix'], specs['versionsuffix']) self.assertEqual(ec['toolchain'], {'name': tcname, 'version': tcver}) self.assertEqual(ec['start_dir'], specs['start_dir']) - os.remove(res[1]) + remove_file(res[1]) specs.update({ 'foo': 'bar123' @@ -608,7 +614,7 @@ def test_obtain_easyconfig(self): self.assertEqual(ec['version'], specs['version']) txt = read_file(res[1]) self.assertTrue(re.search("^version = [\"']%s[\"']$" % ver, txt, re.M)) - os.remove(res[1]) + remove_file(res[1]) # should pick correct toolchain version as well, i.e. now newer than what's specified, if a choice needs to be made specs.update({ @@ -697,6 +703,7 @@ def test_obtain_easyconfig(self): self.assertEqual(res[0], True) error_pattern = "Hidden deps with visible module names .* not in list of \(build\)dependencies: .*" self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, res[1]) + remove_file(res[1]) specs['dependencies'].append(('test', '3.2.1')) @@ -2156,6 +2163,14 @@ def test_get_paths_for(self): res = get_paths_for(subdir='easyconfigs', robot_path=None) self.assertTrue(os.path.samefile(test_ecs, res[0])) + def test_get_module_path(self): + """Test get_module_path function.""" + self.assertEqual(get_module_path('EB_bzip2', generic=False), 'easybuild.easyblocks.bzip2') + self.assertEqual(get_module_path('EB_bzip2'), 'easybuild.easyblocks.bzip2') + + self.assertEqual(get_module_path('RPackage'), 'easybuild.easyblocks.generic.rpackage') + self.assertEqual(get_module_path('RPackage', generic=True), 'easybuild.easyblocks.generic.rpackage') + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/modules.py b/test/framework/modules.py index f4f2daf421..d8943faca4 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -41,8 +41,6 @@ import easybuild.tools.modules as mod from easybuild.framework.easyblock import EasyBlock -from easybuild.framework.easyconfig.easyconfig import EasyConfig -from easybuild.tools import config from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import copy_file, copy_dir, mkdir, read_file, write_file from easybuild.tools.modules import EnvironmentModules, EnvironmentModulesTcl, Lmod, NoModulesTool @@ -240,6 +238,19 @@ def test_prepend_module_path(self): self.modtool.prepend_module_path(symlink_path) self.assertEqual(modulepath, curr_module_paths()) + # test prepending with high priority + test_path_bis = tempfile.mkdtemp(prefix=self.test_prefix) + test_path_tris = tempfile.mkdtemp(prefix=self.test_prefix) + self.modtool.prepend_module_path(test_path_bis, priority=10000) + self.assertEqual(test_path_bis, curr_module_paths()[0]) + + # check whether prepend with priority actually works (only for Lmod) + if isinstance(self.modtool, Lmod): + self.modtool.prepend_module_path(test_path_tris) + modulepath = curr_module_paths() + self.assertEqual(test_path_bis, modulepath[0]) + self.assertEqual(test_path_tris, modulepath[1]) + def test_ld_library_path(self): """Make sure LD_LIBRARY_PATH is what it should be when loaded multiple modules.""" self.init_testmods() @@ -757,6 +768,25 @@ def test_module_caches(self): self.assertEqual(mod.MODULE_AVAIL_CACHE, {}) self.assertEqual(mod.MODULE_SHOW_CACHE, {}) + def test_module_use(self): + """Test 'module use'.""" + test_dir1 = os.path.join(self.test_prefix, 'one') + test_dir2 = os.path.join(self.test_prefix, 'two') + test_dir3 = os.path.join(self.test_prefix, 'three') + + self.assertFalse(test_dir1 in os.environ.get('MODULEPATH', '')) + self.modtool.use(test_dir1) + self.assertTrue(os.environ.get('MODULEPATH', '').startswith('%s:' % test_dir1)) + + # also test use with high priority + self.modtool.use(test_dir2, priority=10000) + self.assertTrue(os.environ['MODULEPATH'].startswith('%s:' % test_dir2)) + + # check whether prepend with priority actually works (only for Lmod) + if isinstance(self.modtool, Lmod): + self.modtool.use(test_dir3) + self.assertTrue(os.environ['MODULEPATH'].startswith('%s:%s:' % (test_dir2, test_dir3))) + def test_module_use_bash(self): """Test whether effect of 'module use' is preserved when a new bash session is started.""" # this test is here as check for a nasty bug in how the modules tool is deployed @@ -814,6 +844,21 @@ def test_load_in_hierarchy(self): self.modtool.load(['hwloc/1.6.2']) self.assertEqual(os.environ['EBROOTHWLOC'], '/tmp/software/Compiler/GCC/4.7.2/hwloc/1.6.2') + # also test whether correct temporary module is loaded even though same module file already exists elsewhere + # with Lmod, this requires prepending the temporary module path to $MODULEPATH with high priority + tmp_moddir = os.path.join(self.test_prefix, 'tmp_modules') + hwloc_mod = os.path.join(tmp_moddir, 'hwloc', '1.6.2') + hwloc_mod_txt = '\n'.join([ + '#%Module', + "module load GCC/4.7.2", + "setenv EBROOTHWLOC /path/to/tmp/hwloc-1.6.2", + ]) + write_file(hwloc_mod, hwloc_mod_txt) + self.modtool.purge() + self.modtool.use(tmp_moddir, priority=10000) + self.modtool.load(['hwloc/1.6.2']) + self.assertTrue(os.environ['EBROOTHWLOC'], "/path/to/tmp/hwloc-1.6.2") + def test_exit_code_check(self): """Verify that EasyBuild checks exit code of executed module commands""" if isinstance(self.modtool, Lmod): diff --git a/test/framework/suite.py b/test/framework/suite.py index 976e2edd97..3e9687ff81 100755 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -83,10 +83,18 @@ # set plain text key ring to be used, # so a GitHub token stored in it can be obtained without having to provide a password try: + # with recent versions of keyring, PlaintextKeyring comes from keyrings.alt import keyring - keyring.set_keyring(keyring.backends.file.PlaintextKeyring()) -except (ImportError, AttributeError): - pass + from keyrings.alt.file import PlaintextKeyring + keyring.set_keyring(PlaintextKeyring()) +except ImportError as err: + try: + # with old versions of keyring, PlaintextKeyring comes from keyring.backends + import keyring + from keyring.backends.file import PlaintextKeyring + keyring.set_keyring(PlaintextKeyring()) + except ImportError as err: + pass # disable all logging to significantly speed up tests fancylogger.disableDefaultHandlers() diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 6ef3767c9d..a90b3f5b6b 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1574,6 +1574,41 @@ def test_toy_sanity_check_commands(self): self.assertTrue(os.path.exists(toy_modfile)) + def test_sanity_check_paths_lib64(self): + """Test whether fallback in sanity check for lib64/ equivalents of library files works.""" + test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs') + ec_file = os.path.join(test_ecs_dir, 'test_ecs', 't', 'toy', 'toy-0.0.eb') + ectxt = read_file(ec_file) + + # modify test easyconfig: move lib/libtoy.a to lib64/libtoy.a + ectxt = re.sub("\s*'files'.*", "'files': ['bin/toy', ('lib/libtoy.a', 'lib/libfoo.a')],", ectxt) + postinstallcmd = "mkdir %(installdir)s/lib64 && mv %(installdir)s/lib/libtoy.a %(installdir)s/lib64/libtoy.a" + ectxt = re.sub("postinstallcmds.*", "postinstallcmds = ['%s']" % postinstallcmd, ectxt) + + test_ec = os.path.join(self.test_prefix, 'toy-0.0.eb') + write_file(test_ec, ectxt) + + # sanity check fails if lib64 fallback in sanity check is disabled + error_pattern = r"Sanity check failed: no file found at 'lib/libtoy.a' or 'lib/libfoo.a' in " + self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec, + extra_args=['--disable-lib64-fallback-sanity-check'], raise_error=True, verbose=False) + + # all is fine is lib64 fallback check is enabled (which it is by default) + self.test_toy_build(ec_file=test_ec, raise_error=True) + + # also check other way around (lib64 -> lib) + ectxt = read_file(ec_file) + ectxt = re.sub("\s*'files'.*", "'files': ['bin/toy', 'lib64/libtoy.a'],", ectxt) + write_file(test_ec, ectxt) + + # sanity check fails if lib64 fallback in sanity check is disabled, since lib64/libtoy.a is not there + error_pattern = r"Sanity check failed: no file found at 'lib64/libtoy.a' in " + self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec, + extra_args=['--disable-lib64-fallback-sanity-check'], raise_error=True, verbose=False) + + # sanity check passes when lib64 fallback is enabled (by default), since lib/libtoy.a is also considered + self.test_toy_build(ec_file=test_ec, raise_error=True) + def test_toy_dumped_easyconfig(self): """ Test dumping of file in eb_filerepo in both .eb and .yeb format """ filename = 'toy-0.0' diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 21f855d540..022e7a4a97 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -29,11 +29,13 @@ """ import os import sys -from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner from easybuild.framework.easyconfig.parser import EasyConfigParser from easybuild.framework.easyconfig.tweak import find_matching_easyconfigs, obtain_ec_for, pick_version, tweak_one +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import write_file class TweakTest(EnhancedTestCase): @@ -131,10 +133,23 @@ def test_tweak_one_version(self): tweaked_val = tweaked_toy_ec_parsed.get(key) self.assertEqual(val, tweaked_val, "Different value for %s parameter: %s vs %s" % (key, val, tweaked_val)) + # check behaviour if target file already exists + error_pattern = "A file already exists at .* where tweaked easyconfig file would be written" + self.assertErrorRegex(EasyBuildError, error_pattern, tweak_one, toy_ec, tweaked_toy_ec, {'version': '1.2.3'}) + + # existing file does get overwritten when --force is used + build_options = {'force': True} + init_config(build_options=build_options) + write_file(tweaked_toy_ec, '') + tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'}) + tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict() + self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3') + def suite(): """ return all the tests in this file """ return TestLoaderFiltered().loadTestsFromTestCase(TweakTest, sys.argv[1:]) + if __name__ == '__main__': TextTestRunner(verbosity=1).run(suite())