diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 13ef17e9cc..241f8b3d8c 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -3,6 +3,131 @@ For more detailed information, please see the git log. These release notes can also be consulted at http://easybuild.readthedocs.org/en/latest/Release_notes.html. +v2.1.1 (May 18th 2015) +---------------------- + +bugfix release +- fix issue with missing load statements when --module-only is used, don't skip ready/prepare steps (#1276) +- enhance --search: only consider actual filename (not entire path), use regex syntax (#1281) +- various other bug fixes, including: + - fix generate_software_list.py script w.r.t. dependencies marked as external modules (#1273) + - only use $LMOD_CMD value if lmod binary can't be found in $PATH (#1275) + - fix location of module_only build option w.r.t. default value (#1277) + - fix combined use of --hide-deps and hiddendependencies (#1280) + - remove log handlers that were added during tests, to ensure effective cleanup of log files (#1282) + - this makes the unit test suite run ~3x faster! + - define $CRAYPE_LINK_TYPE if 'dynamic' toolchain option is enabled for Cray compiler wrappers (#1283) + +v2.1.0 (April 30th 2015) +------------------------ + +feature + bugfix release +- requires vsc-base v2.2.0 or more recent + - added support for LoggedException + - added support for add_flex action in GeneralOption + - added support to GeneralOption to act on unknown configuration environment variables +- add support for only (re)generating module files: --module-only (#1018) + - module naming scheme API is enhanced to include det_install_subdir method + - see http://easybuild.readthedocs.org/en/latest/Partial_installations.html#module-only +- add support for generating module files in Lua syntax (note: requires Lmod as modules tool) (#1060, #1255, #1256, #1270) + - see --module-syntax configuration option and http://easybuild.readthedocs.org/en/latest/Configuration.html#module-syntax +- deprecate log.error method in favor of raising EasyBuildError exception (#1218) + - see http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html#depr-error-reporting +- add support for using external modules as dependencies, and to provide metadata for external modules (#1230, #1265, #1267) + - see http://easybuild.readthedocs.org/en/latest/Using_external_modules.html +- add experimental support for Cray toolchains on top of PrgEnv modules: CrayGNU, CrayIntel, CrayCCE (#1234, #1268) + - see https://github.com/hpcugent/easybuild/wiki/EasyBuild-on-Cray for more information +- various other enhancements, including: + - clear list of checksums when using --try-software-version (#1169) + - sort the results of searching for files (e.g., --search output) (#1214) + - enhance test w.r.t. use of templates in cfgfile (#1217) + - define %(DEFAULT_REPOSITORYPATH)s template for cfgfiles (see eb --avail-cfgfile-constants) (#1220) + - also reset $LD_PRELOAD when running module commands, in case module defined $LD_PRELOAD (#1222) + - move location of 'module use' statements in generated module file (*after* 'module load' statements) (#1232) + - add support for --show-default-configfiles (#1240) + - see http://easybuild.readthedocs.org/en/latest/Configuration.html#default-configuration-files + - report error on missing configuration files, rather than ignoring them (#1240) + - see http://easybuild.readthedocs.org/en/latest/Configuration.html#configuration-env-vars + - clean up commit message used in easyconfig git repository (#1248) + - add --hide-deps configuration option to specify names of software that must be installed as hidden modules (#1250) + - see http://easybuild.readthedocs.org/en/latest/Manipulating_dependencies.html#hide-deps + - add support for appending/prepending to --robot-paths to avoid overwriting default robot search path (#1252) + - see also http://easybuild.readthedocs.org/en/latest/Using_the_EasyBuild_command_line.html#robot-search-path-prepend-append + - enable detection of use of unknown $EASYBUILD-prefixed environment variables (#1253) + - add --installpath-modules and --installpath-software configuration options (#1258) + - see http://easybuild.readthedocs.org/en/latest/Configuration.html#installpath + - use dedicated subdirectory in temporary directory for each test to ensure better cleanup (#1260) + - get rid of $PROFILEREAD hack when running commands, not needed anymore (#1264) +- various bug fixes, including: + - make bootstrap script robust against having vsc-base already available in Python search path (#1212, #1215) + - set default value for unpack_options easyconfig parameter to '', so self.cfg.update works on it (#1229) + - also copy rotated log files (#1238) + - fix parsing of --download-timeout value (#1242) + - make test_XDG_CONFIG_env_vars unit test robust against existing user config file in default location (#1259) + - fix minor robustness issues w.r.t. $XDG_CONFIG* and $PYTHONPATH in unit tests (#1262) + - fix issue with handling empty toolchain variables (#1263) + +v2.0.0 (March 6th 2015) +----------------------- + +feature + bugfix release +- requires vsc-base v2.0.3 or more recent + - avoid deprecation warnings w.r.t. use of 'message' attribute (hpcugent/vsc-base#155) + - fix typo in log message rendering --ignoreconfigfiles unusable (hpcugent/vsc-base#158) +- removed functionality that was deprecated for EasyBuild version 2.0 (#1143) + - see http://easybuild.readthedocs.org/en/latest/Removed-functionality.html + - the fix_broken_easyconfigs.py script can be used to update easyconfig files suffering from this (#1151, #1206, #1207) + - for more information about this script, see http://easybuild.readthedocs.org/en/latest/Useful-scripts.html#fix-broken-easyconfigs-py +- stop including a crippled copy of vsc-base, include vsc-base as a proper dependency instead (#1160, #1194) + - vsc-base is automatically installed as a dependency for easybuild-framework, if a Python installation tool is used + - see http://easybuild.readthedocs.org/en/latest/Installation.html#required-python-packages +- various other enhancements, including: + - add support for Linux/POWER systems (#1044) + - major cleanup in tools/systemtools.py + significantly enhanced tests (#1044) + - add support for 'eb -a rst', list available easyconfig parameters in ReST format (#1131) + - add support for specifying one or more easyconfigs in combination with --from-pr (#1132) + - see http://easybuild.readthedocs.org/en/latest/Integration_with_GitHub.html#using-easyconfigs-from-pull-requests-via-from-pr + - define __contains__ in EasyConfig class (#1155) + - restore support for downloading over a proxy (#1158) + - i.e., use urllib2 rather than urllib + - this involved sacrificing the download progress report (which was only visible in the log file) + - let mpi_family return None if MPI is not supported by a toolchain (#1164) + - include support for specifying system-level configuration files for EasyBuild via $XDG_CONFIG_DIRS (#1166) + - see http://easybuild.readthedocs.org/en/latest/Configuration.html#default-configuration-files + - make unit tests more robust (#1167, #1196) + - see http://easybuild.readthedocs.org/en/latest/Unit-tests.html + - add hierarchical module naming scheme categorizing modules by 'moduleclass' (#1176) + - enhance bootstrap script to allow bootstrapping using supplied tarballs (#1184) + - see http://easybuild.readthedocs.org/en/latest/Installation.html#advanced-bootstrapping-options + - disable updating of Lmod user cache by default, add configuration option --update-modules-tool-cache (#1185) + - for now, only the Lmod user cache can be updated using --update-modules-tool-cache + - use available which() function, rather than running 'which' via run_cmd (#1192) + - fix install-EasyBuild-develop.sh script w.r.t. vsc-base dependency (#1193) + - also consider robot search path when looking for specified easyconfigs (#1201) + - see http://easybuild.readthedocs.org/en/latest/Using_the_EasyBuild_command_line.html#specifying-easyconfigs +- various bug fixes, including: + - stop triggering deprecated/no longer support functionality in unit tests (#1126) + - fix from_pr test by including dummy easyblocks for HPL and ScaLAPACK (#1133) + - escape use of '%' in string with command line options with --job (#1135) + - fix handling specified patch level 0 (+ enhance tests for fetch_patches method) (#1139) + - fix formatting issues in generated easyconfig file obtained via --try-X (#1144) + - use log.error in tools/toolchain/toolchain.py where applicable (#1145) + - stop hardcoding /tmp in mpi_cmd_for function (#1146, #1200) + - correctly determine variable name for $EBEXTLIST when generating module file (#1156) + - do not ignore exit code of failing postinstall commands (#1157) + - fix rare case in which used easyconfig and copied easyconfig are the same (#1159) + - always filter hidden deps from list of dependencies (#1161) + - fix implementation of path_matches function in tools/filetools.py (#1163) + - make sure plain text keyring is used by unit tests (#1165) + - suppress creation of module symlinks for HierarchicalMNS (#1173) + - sort all lists obtained via glob.glob, since they are in arbitrary order (#1187) + - stop modifying $MODULEPATH directly in setUp/tearDown of toolchain tests (#1191) + +v1.16.2 (March 6th 2015) +------------------------ + +(no changes compared to v1.16.1, simple version bump to stay in sync with easybuild-easyblocks) + v1.16.1 (December 19th 2014) ---------------------------- diff --git a/easybuild/__init__.py b/easybuild/__init__.py index 2b217b34bc..6bdfdf13a4 100644 --- a/easybuild/__init__.py +++ b/easybuild/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2014 Ghent University +# Copyright 2011-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/__init__.py b/easybuild/framework/__init__.py index 20b1669d93..bd67a8111f 100644 --- a/easybuild/framework/__init__.py +++ b/easybuild/framework/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 008857c8ac..3c03555429 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -63,12 +63,12 @@ from easybuild.tools.environment import restore_env from easybuild.tools.filetools import DEFAULT_CHECKSUM from easybuild.tools.filetools import adjust_permissions, apply_patch, convert_name, download_file, encode_class_name -from easybuild.tools.filetools import extract_file, mkdir, read_file, rmtree2 +from easybuild.tools.filetools import extract_file, mkdir, move_logs, read_file, rmtree2 from easybuild.tools.filetools import write_file, compute_checksum, verify_checksum from easybuild.tools.run import run_cmd from easybuild.tools.jenkins import write_to_xml -from easybuild.tools.module_generator import ModuleGenerator -from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version +from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator +from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX from easybuild.tools.modules import get_software_root, modules_tool from easybuild.tools.repository.repository import init_repository @@ -78,6 +78,25 @@ from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION +BUILD_STEP = 'build' +CLEANUP_STEP = 'cleanup' +CONFIGURE_STEP = 'configure' +EXTENSIONS_STEP = 'extensions' +FETCH_STEP = 'fetch' +MODULE_STEP = 'module' +PACKAGE_STEP = 'package' +PATCH_STEP = 'patch' +POSTPROC_STEP = 'postproc' +PREPARE_STEP = 'prepare' +READY_STEP = 'ready' +SANITYCHECK_STEP = 'sanitycheck' +SOURCE_STEP = 'source' +TEST_STEP = 'test' +TESTCASES_STEP = 'testcases' + +MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP] + + _log = fancylogger.getLogger('easyblock') @@ -132,7 +151,7 @@ def __init__(self, ec): # modules interface with default MODULEPATH self.modules_tool = modules_tool() # module generator - self.module_generator = ModuleGenerator(self, fake=True) + self.module_generator = module_generator(self, fake=True) # modules footer self.modules_footer = None @@ -144,7 +163,7 @@ def __init__(self, ec): if isinstance(ec, EasyConfig): self.cfg = ec else: - _log.error("Value of incorrect type passed to EasyBlock constructor: %s ('%s')" % (type(ec), ec)) + raise EasyBuildError("Value of incorrect type passed to EasyBlock constructor: %s ('%s')", type(ec), ec) # determine install subdirectory, based on module name self.install_subdir = None @@ -158,9 +177,6 @@ def __init__(self, ec): self.logdebug = build_option('debug') self.postmsg = '' # allow a post message to be set, which can be shown as last output - # original environ will be set later - self.orig_environ = {} - # list of loaded modules self.loaded_modules = [] @@ -176,8 +192,8 @@ def __init__(self, ec): # original module path self.orig_modulepath = os.getenv('MODULEPATH') - # keep track of original environment, so we restore it if needed - self.orig_environ = copy.deepcopy(os.environ) + # keep track of initial environment we start in, so we can restore it if needed + self.initial_environ = copy.deepcopy(os.environ) # initialize logger self._init_log() @@ -217,8 +233,8 @@ def _init_log(self): self.log.info(this_is_easybuild()) this_module = inspect.getmodule(self) - tup = (self.__class__.__name__, this_module.__name__, this_module.__file__) - self.log.info("This is easyblock %s from module %s (%s)" % tup) + self.log.info("This is easyblock %s from module %s (%s)", + self.__class__.__name__, this_module.__name__, this_module.__file__) def close_log(self): """ @@ -247,7 +263,7 @@ def get_checksum_for(self, checksums, filename=None, index=None): elif checksums is None: return None else: - self.log.error("Invalid type for checksums (%s), should be list, tuple or None." % type(checksums)) + raise EasyBuildError("Invalid type for checksums (%s), should be list, tuple or None.", type(checksums)) def fetch_sources(self, list_of_sources, checksums=None): """ @@ -276,7 +292,7 @@ def fetch_sources(self, list_of_sources, checksums=None): 'finalpath': self.builddir, }) else: - self.log.error('No file found for source %s' % source) + raise EasyBuildError('No file found for source %s', source) self.log.info("Added sources: %s" % self.src) @@ -297,8 +313,8 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None): level = None if isinstance(patch_spec, (list, tuple)): if not len(patch_spec) == 2: - self.log.error("Unknown patch specification '%s', only two-element lists/tuples are supported!", - str(patch_spec)) + raise EasyBuildError("Unknown patch specification '%s', only 2-element lists/tuples are supported!", + str(patch_spec)) patch_file = patch_spec[0] # this *must* be of typ int, nothing else @@ -311,7 +327,8 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None): copy_file = True suff = patch_spec[1] else: - self.log.error("Wrong patch spec '%s', only int/string are supported as 2nd element" % str(patch_spec)) + raise EasyBuildError("Wrong patch spec '%s', only int/string are supported as 2nd element", + str(patch_spec)) else: patch_file = patch_spec @@ -336,7 +353,7 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None): else: self.patches.append(patchspec) else: - self.log.error('No file found for patch %s' % patch_spec) + raise EasyBuildError('No file found for patch %s', patch_spec) if extension: self.log.info("Fetched extension patches: %s" % patches) @@ -370,9 +387,9 @@ def fetch_extension_sources(self): ext_options = ext[2] if not isinstance(ext_options, dict): - self.log.error("Unexpected type (non-dict) for 3rd element of %s" % ext) + raise EasyBuildError("Unexpected type (non-dict) for 3rd element of %s", ext) elif len(ext) > 3: - self.log.error('Extension specified in unknown format (list/tuple too long)') + raise EasyBuildError('Extension specified in unknown format (list/tuple too long)') ext_src = { 'name': ext_name, @@ -401,7 +418,7 @@ def fetch_extension_sources(self): if verify_checksum(src_fn, fn_checksum): self.log.info('Checksum for ext source %s verified' % fn) else: - self.log.error('Checksum for ext source %s failed' % fn) + raise EasyBuildError('Checksum for ext source %s failed', fn) ext_patches = self.fetch_patches(patch_specs=ext_options.get('patches', []), extension=True) if ext_patches: @@ -415,20 +432,20 @@ def fetch_extension_sources(self): if verify_checksum(ext_patch, checksum): self.log.info('Checksum for extension patch %s verified' % ext_patch) else: - self.log.error('Checksum for extension patch %s failed' % ext_patch) + raise EasyBuildError('Checksum for extension patch %s failed', ext_patch) else: self.log.debug('No patches found for extension %s.' % ext_name) exts_sources.append(ext_src) else: - self.log.error("Source for extension %s not found.") + raise EasyBuildError("Source for extension %s not found.", ext) elif isinstance(ext, basestring): exts_sources.append({'name': ext}) else: - self.log.error("Extension specified in unknown format (not a string/list/tuple)") + raise EasyBuildError("Extension specified in unknown format (not a string/list/tuple)") return exts_sources @@ -468,7 +485,7 @@ def obtain_file(self, filename, extension=False, urls=None): return fullpath except IOError, err: - self.log.exception("Downloading file %s from url %s to %s failed: %s" % (filename, url, fullpath, err)) + raise EasyBuildError("Downloading file %s from url %s to %s failed: %s", filename, url, fullpath, err) else: # try and find file in various locations @@ -573,7 +590,8 @@ def obtain_file(self, filename, extension=False, urls=None): else: failedpaths.append(fullurl) - self.log.error("Couldn't find file %s anywhere, and downloading it didn't work either...\nPaths attempted (in order): %s " % (filename, ', '.join(failedpaths))) + raise EasyBuildError("Couldn't find file %s anywhere, and downloading it didn't work either... " + "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) # # GETTER/SETTER UTILITY FUNCTIONS @@ -652,8 +670,8 @@ def make_builddir(self): if not self.build_in_installdir: # self.builddir should be already set by gen_builddir() if not self.builddir: - self.log.error("self.builddir not set, make sure gen_builddir() is called first!") - self.log.debug("Creating the build directory %s (cleanup: %s)" % (self.builddir, self.cfg['cleanupoldbuild'])) + raise EasyBuildError("self.builddir not set, make sure gen_builddir() is called first!") + self.log.debug("Creating the build directory %s (cleanup: %s)", self.builddir, self.cfg['cleanupoldbuild']) else: self.log.info("Changing build dir to %s" % self.installdir) self.builddir = self.installdir @@ -675,12 +693,11 @@ def gen_installdir(self): """ basepath = install_path() if basepath: - self.install_subdir = ActiveMNS().det_full_module_name(self.cfg, force_visible=True) - installdir = os.path.join(basepath, self.install_subdir) - self.installdir = os.path.abspath(installdir) + self.install_subdir = ActiveMNS().det_install_subdir(self.cfg) + self.installdir = os.path.join(os.path.abspath(basepath), self.install_subdir) self.log.info("Install dir set to %s" % self.installdir) else: - self.log.error("Can't set installation directory") + raise EasyBuildError("Can't set installation directory") def make_installdir(self, dontcreate=None): """ @@ -707,7 +724,7 @@ def make_dir(self, dir_name, clean, dontcreateinstalldir=False): rmtree2(dir_name) self.log.info("Removed old directory %s" % dir_name) except OSError, err: - self.log.exception("Removal of old directory %s failed: %s" % (dir_name, err)) + raise EasyBuildError("Removal of old directory %s failed: %s", dir_name, err) else: try: timestamp = time.strftime("%Y%m%d-%H%M%S") @@ -715,7 +732,7 @@ def make_dir(self, dir_name, clean, dontcreateinstalldir=False): shutil.move(dir_name, backupdir) self.log.info("Moved old directory %s to %s" % (dir_name, backupdir)) except OSError, err: - self.log.exception("Moving old directory to backup %s %s failed: %s" % (dir_name, backupdir, err)) + raise EasyBuildError("Moving old directory to backup %s %s failed: %s", dir_name, backupdir, err) if dontcreateinstalldir: olddir = dir_name @@ -745,17 +762,11 @@ def make_devel_module(self, create_in_builddir=False): # load fake module fake_mod_data = self.load_fake_module(purge=True) - mod_gen = ModuleGenerator(self) - header = "#%Module\n" - - env_txt = "" - for (key, val) in env.get_changes().items(): - # check if non-empty string - # TODO: add unset for empty vars? - if val.strip(): - env_txt += mod_gen.set_environment(key, val) + header = self.module_generator.MODULE_HEADER + if header: + header += '\n' - load_txt = "" + load_lines = [] # capture all the EBDEVEL vars # these should be all the dependencies and we should load them for key in os.environ: @@ -765,10 +776,17 @@ def make_devel_module(self, create_in_builddir=False): path = os.environ[key] if os.path.isfile(path): mod_name = path.rsplit(os.path.sep, 1)[-1] - load_txt += mod_gen.load_module(mod_name) + load_lines.append(self.module_generator.load_module(mod_name)) elif key.startswith('SOFTDEVEL'): self.log.nosupport("Environment variable SOFTDEVEL* being relied on", '2.0') + env_lines = [] + for (key, val) in env.get_changes().items(): + # check if non-empty string + # TODO: add unset for empty vars? + if val.strip(): + env_lines.append(self.module_generator.set_environment(key, val)) + if create_in_builddir: output_dir = self.builddir else: @@ -778,7 +796,8 @@ def make_devel_module(self, create_in_builddir=False): filename = os.path.join(output_dir, ActiveMNS().det_devel_module_filename(self.cfg)) self.log.debug("Writing devel module to %s" % filename) - write_file(filename, header + load_txt + env_txt) + txt = ''.join([header] + load_lines + env_lines) + write_file(filename, txt) # cleanup: unload fake module, remove fake module dir self.clean_up_fake_module(fake_mod_data) @@ -842,32 +861,54 @@ def make_module_extra(self): """ Sets optional variables (EBROOT, MPI tuning variables). """ - txt = "\n" + lines = [''] # EBROOT + EBVERSION + EBDEVEL - environment_name = convert_name(self.name, upper=True) - txt += self.module_generator.set_environment(ROOT_ENV_VAR_NAME_PREFIX + environment_name, "$root") - txt += self.module_generator.set_environment(VERSION_ENV_VAR_NAME_PREFIX + environment_name, self.version) - devel_path = os.path.join("$root", log_path(), ActiveMNS().det_devel_module_filename(self.cfg)) - txt += self.module_generator.set_environment(DEVEL_ENV_VAR_NAME_PREFIX + environment_name, devel_path) + env_name = convert_name(self.name, upper=True) - txt += "\n" + lines.append(self.module_generator.set_environment(ROOT_ENV_VAR_NAME_PREFIX + env_name, '', relpath=True)) + lines.append(self.module_generator.set_environment(VERSION_ENV_VAR_NAME_PREFIX + env_name, self.version)) + + devel_path = os.path.join(log_path(), ActiveMNS().det_devel_module_filename(self.cfg)) + devel_path_envvar = DEVEL_ENV_VAR_NAME_PREFIX + env_name + lines.append(self.module_generator.set_environment(devel_path_envvar, devel_path, relpath=True)) + + lines.append('\n') for (key, value) in self.cfg['modextravars'].items(): - txt += self.module_generator.set_environment(key, value) + lines.append(self.module_generator.set_environment(key, value)) + for (key, value) in self.cfg['modextrapaths'].items(): if isinstance(value, basestring): value = [value] elif not isinstance(value, (tuple, list)): - self.log.error("modextrapaths dict value %s (type: %s) is not a list or tuple" % (value, type(value))) - txt += self.module_generator.prepend_paths(key, value) + raise EasyBuildError("modextrapaths dict value %s (type: %s) is not a list or tuple", + value, type(value)) + lines.append(self.module_generator.prepend_paths(key, value)) + if self.cfg['modloadmsg']: - txt += self.module_generator.msg_on_load(self.cfg['modloadmsg']) + lines.append(self.module_generator.msg_on_load(self.cfg['modloadmsg'])) + if self.cfg['modtclfooter']: - txt += self.module_generator.add_tcl_footer(self.cfg['modtclfooter']) + if isinstance(self.module_generator, ModuleGeneratorTcl): + self.log.debug("Including Tcl footer in module: %s", self.cfg['modtclfooter']) + lines.extend([self.cfg['modtclfooter'], '\n']) + else: + self.log.warning("Not including footer in Tcl syntax in non-Tcl module file: %s", + self.cfg['modtclfooter']) + + if self.cfg['modluafooter']: + if isinstance(self.module_generator, ModuleGeneratorLua): + self.log.debug("Including Lua footer in module: %s", self.cfg['modluafooter']) + lines.extend([self.cfg['modluafooter'], '\n']) + else: + self.log.warning("Not including footer in Lua syntax in non-Lua module file: %s", + self.cfg['modluafooter']) + for (key, value) in self.cfg['modaliases'].items(): - txt += self.module_generator.set_alias(key, value) + lines.append(self.module_generator.set_alias(key, value)) - self.log.debug("make_module_extra added this: %s" % txt) + txt = ''.join(lines) + self.log.debug("make_module_extra added this: %s", txt) return txt @@ -876,32 +917,32 @@ def make_module_extra_extensions(self): Sets optional variables for extensions. """ # add stuff specific to individual extensions - txt = self.module_extra_extensions + lines = [self.module_extra_extensions] # set environment variable that specifies list of extensions if self.exts_all: exts_list = ','.join(['%s-%s' % (ext['name'], ext.get('version', '')) for ext in self.exts_all]) env_var_name = convert_name(self.name, upper=True) - txt += self.module_generator.set_environment('EBEXTSLIST%s' % env_var_name, exts_list) + lines.append(self.module_generator.set_environment('EBEXTSLIST%s' % env_var_name, exts_list)) - return txt + return ''.join(lines) def make_module_footer(self): """ Insert a footer section in the modulefile, primarily meant for contextual information """ - txt = '\n# Built with EasyBuild version %s\n' % VERBOSE_VERSION + footer = [self.module_generator.comment("Built with EasyBuild version %s" % VERBOSE_VERSION)] # add extra stuff for extensions (if any) if self.cfg['exts_list']: - txt += self.make_module_extra_extensions() + footer.append(self.make_module_extra_extensions()) # include modules footer if one is specified if self.modules_footer is not None: self.log.debug("Including specified footer into module: '%s'" % self.modules_footer) - txt += self.modules_footer + footer.append(self.modules_footer) - return txt + return ''.join(footer) def make_module_extend_modpath(self): """ @@ -928,25 +969,25 @@ def make_module_req(self): """ requirements = self.make_module_req_guess() - if os.path.exists(self.installdir): + lines = [] + if os.path.isdir(self.installdir): try: os.chdir(self.installdir) except OSError, err: - self.log.error("Failed to change to %s: %s" % (self.installdir, err)) + raise EasyBuildError("Failed to change to %s: %s", self.installdir, err) - txt = "\n" + lines.append('\n') for key in sorted(requirements): for path in requirements[key]: paths = sorted(glob.glob(path)) if paths: - txt += self.module_generator.prepend_paths(key, paths) + lines.append(self.module_generator.prepend_paths(key, paths)) try: os.chdir(self.orig_workdir) except OSError, err: - self.log.error("Failed to change back to %s: %s" % (self.orig_workdir, err)) - else: - txt = "" - return txt + raise EasyBuildError("Failed to change back to %s: %s", self.orig_workdir, err) + + return ''.join(lines) def make_module_req_guess(self): """ @@ -973,7 +1014,7 @@ def load_module(self, mod_paths=None, purge=True): mod_paths = [] all_mod_paths = mod_paths + ActiveMNS().det_init_modulepaths(self.cfg) mods = [self.full_mod_name] - self.modules_tool.load(mods, mod_paths=all_mod_paths, purge=purge, orig_env=self.orig_environ) + self.modules_tool.load(mods, mod_paths=all_mod_paths, purge=purge, init_env=self.initial_environ) else: self.log.warning("Not loading module, since self.full_mod_name is not set.") @@ -981,9 +1022,8 @@ def load_fake_module(self, purge=False): """ Create and load fake module. """ - - # take a copy of the environment before loading the fake module, so we can restore it - orig_env = copy.deepcopy(os.environ) + # take a copy of the current environment before loading the fake module, so we can restore it + env = copy.deepcopy(os.environ) # create fake module fake_mod_path = self.make_module_step(True) @@ -993,13 +1033,13 @@ def load_fake_module(self, purge=False): modtool.prepend_module_path(fake_mod_path) self.load_module(purge=purge) - return (fake_mod_path, orig_env) + return (fake_mod_path, env) def clean_up_fake_module(self, fake_mod_data): """ Clean up fake module. """ - fake_mod_path, orig_env = fake_mod_data + fake_mod_path, env = fake_mod_data # unload module and remove temporary module directory # self.full_mod_name might not be set (e.g. during unit tests) if fake_mod_path and self.full_mod_name is not None: @@ -1009,12 +1049,12 @@ def clean_up_fake_module(self, fake_mod_data): modtool.remove_module_path(fake_mod_path) rmtree2(os.path.dirname(fake_mod_path)) except OSError, err: - self.log.error("Failed to clean up fake module dir %s: %s" % (fake_mod_path, err)) + raise EasyBuildError("Failed to clean up fake module dir %s: %s", fake_mod_path, err) elif self.full_mod_name is None: self.log.warning("Not unloading module, since self.full_mod_name is not set.") # restore original environment - restore_env(orig_env) + restore_env(env) def load_dependency_modules(self): """Load dependency modules.""" @@ -1042,9 +1082,9 @@ def skip_extensions(self): self.cfg.enable_templating = True if not exts_filter or len(exts_filter) == 0: - self.log.error("Skipping of extensions, but no exts_filter set in easyconfig") + raise EasyBuildError("Skipping of extensions, but no exts_filter set in easyconfig") elif isinstance(exts_filter, basestring) or len(exts_filter) != 2: - self.log.error('exts_filter should be a list or tuple of ("command","input")') + raise EasyBuildError('exts_filter should be a list or tuple of ("command","input")') cmdtmpl = exts_filter[0] cmdinputtmpl = exts_filter[1] if not self.exts: @@ -1110,7 +1150,7 @@ def guess_start_dir(self): os.chdir(self.cfg['start_dir']) self.log.debug("Changed to real build directory %s" % (self.cfg['start_dir'])) except OSError, err: - self.log.exception("Can't change to real build directory %s: %s" % (self.cfg['start_dir'], err)) + raise EasyBuildError("Can't change to real build directory %s: %s", self.cfg['start_dir'], err) def handle_iterate_opts(self): """Handle options relevant during iterated part of build/install procedure.""" @@ -1164,12 +1204,12 @@ def check_readiness_step(self): if not len(self.cfg.dependencies()) == len(self.toolchain.dependencies): self.log.debug("dep %s (%s)" % (len(self.cfg.dependencies()), self.cfg.dependencies())) self.log.debug("tc.dep %s (%s)" % (len(self.toolchain.dependencies), self.toolchain.dependencies)) - self.log.error('Not all dependencies have a matching toolchain version') + raise EasyBuildError('Not all dependencies have a matching toolchain version') # check if the application is not loaded at the moment (root, env_var) = get_software_root(self.name, with_env_var=True) if root: - self.log.error("Module is already loaded (%s is set), installation cannot continue." % env_var) + raise EasyBuildError("Module is already loaded (%s is set), installation cannot continue.", env_var) # check if main install needs to be skipped # - if a current module can be found, skip is ok @@ -1189,12 +1229,15 @@ def fetch_step(self, skip_checksums=False): # check EasyBuild version easybuild_version = self.cfg['easybuild_version'] if not easybuild_version: - self.log.warn("Easyconfig does not specify an EasyBuild-version (key 'easybuild_version')! Assuming the latest version") + self.log.warn("Easyconfig does not specify an EasyBuild-version (key 'easybuild_version')! " + "Assuming the latest version") else: if LooseVersion(easybuild_version) < VERSION: - self.log.warn("EasyBuild-version %s is older than the currently running one. Proceed with caution!" % easybuild_version) + self.log.warn("EasyBuild-version %s is older than the currently running one. Proceed with caution!", + easybuild_version) elif LooseVersion(easybuild_version) > VERSION: - self.log.error("EasyBuild-version %s is newer than the currently running one. Aborting!" % easybuild_version) + raise EasyBuildError("EasyBuild-version %s is newer than the currently running one. Aborting!", + easybuild_version) # fetch sources if self.cfg['sources']: @@ -1249,7 +1292,7 @@ def checksum_step(self): for fil in self.src + self.patches: ok = verify_checksum(fil['path'], fil['checksum']) if not ok: - self.log.error("Checksum verification for %s using %s failed." % (fil['path'], fil['checksum'])) + raise EasyBuildError("Checksum verification for %s using %s failed.", fil['path'], fil['checksum']) else: self.log.info("Checksum verification for %s using %s passed." % (fil['path'], fil['checksum'])) @@ -1263,7 +1306,7 @@ def extract_step(self): if srcdir: self.src[self.src.index(src)]['finalpath'] = srcdir else: - self.log.error("Unpacking source %s failed" % src['name']) + raise EasyBuildError("Unpacking source %s failed", src['name']) def patch_step(self, beginpath=None): """ @@ -1281,16 +1324,16 @@ def patch_step(self, beginpath=None): # determine whether 'patch' file should be copied rather than applied copy_patch = 'copy' in patch and not 'sourcepath' in patch - tup = (srcind, level, srcpathsuffix, copy) - self.log.debug("Source index: %s; patch level: %s; source path suffix: %s; copy patch: %s" % tup) + self.log.debug("Source index: %s; patch level: %s; source path suffix: %s; copy patch: %s", + srcind, level, srcpathsuffix, copy) if beginpath is None: try: beginpath = self.src[srcind]['finalpath'] self.log.debug("Determine begin path for patch %s: %s" % (patch['name'], beginpath)) except IndexError, err: - tup = (patch['name'], srcind, self.src, err) - self.log.error("Can't apply patch %s to source at index %s of list %s: %s" % tup) + raise EasyBuildError("Can't apply patch %s to source at index %s of list %s: %s", + patch['name'], srcind, self.src, err) else: self.log.debug("Using specified begin path for patch %s: %s" % (patch['name'], beginpath)) @@ -1298,14 +1341,19 @@ def patch_step(self, beginpath=None): self.log.debug("Applying patch %s in path %s" % (patch, src)) if not apply_patch(patch['path'], src, copy=copy_patch, level=level): - self.log.error("Applying patch %s failed" % patch['name']) + raise EasyBuildError("Applying patch %s failed", patch['name']) def prepare_step(self): """ Pre-configure step. Set's up the builddir just before starting configure """ + # clean environment, undefine any unwanted environment variables that may be harmful self.cfg['unwanted_env_vars'] = env.unset_env_vars(self.cfg['unwanted_env_vars']) + + # prepare toolchain: load toolchain module and dependencies, set up build environment self.toolchain.prepare(self.cfg['onlytcmod']) + + # guess directory to start configure/build/install process in, and move there self.guess_start_dir() def configure_step(self): @@ -1368,7 +1416,7 @@ def extensions_step(self, fetch=False): # we really need a default class if not exts_defaultclass: self.clean_up_fake_module(fake_mod_data) - self.log.error("ERROR: No default extension class set for %s" % self.name) + raise EasyBuildError("ERROR: No default extension class set for %s", self.name) # obtain name and module path for default extention class if hasattr(exts_defaultclass, '__iter__'): @@ -1380,7 +1428,7 @@ def extensions_step(self, fetch=False): default_class_modpath = get_module_path(default_class, generic=True) else: - self.log.error("Improper default extension class specification, should be list/tuple or string.") + raise EasyBuildError("Improper default extension class specification, should be list/tuple or string.") # get class instances for all extensions for ext in self.exts: @@ -1413,7 +1461,8 @@ def extensions_step(self, fetch=False): cls = get_class_for(mod_path, class_name) inst = cls(self, ext) except (ImportError, NameError), err: - self.log.error("Failed to load specified class %s for extension %s: %s" % (class_name, ext['name'], err)) + raise EasyBuildError("Failed to load specified class %s for extension %s: %s", + class_name, ext['name'], err) # fallback attempt: use default class if inst is None: @@ -1421,12 +1470,11 @@ def extensions_step(self, fetch=False): cls = get_class_for(default_class_modpath, default_class) self.log.debug("Obtained class %s for installing extension %s" % (cls, ext['name'])) inst = cls(self, ext) - tup = (ext['name'], default_class, default_class_modpath) - self.log.debug("Installing extension %s with default class %s (from %s)" % tup) + self.log.debug("Installing extension %s with default class %s (from %s)", + ext['name'], default_class, default_class_modpath) except (ImportError, NameError), err: - msg = "Also failed to use default class %s from %s for extension %s: %s, giving up" % \ - (default_class, default_class_modpath, ext['name'], err) - self.log.error(msg) + raise EasyBuildError("Also failed to use default class %s from %s for extension %s: %s, giving up", + default_class, default_class_modpath, ext['name'], err) else: self.log.debug("Installing extension %s with class %s (from %s)" % (ext['name'], class_name, mod_path)) @@ -1457,10 +1505,10 @@ def post_install_step(self): if self.cfg['postinstallcmds'] is not None: # make sure we have a list of commands if not isinstance(self.cfg['postinstallcmds'], (list, tuple)): - self.log.error("Invalid value for 'postinstallcmds', should be list or tuple of strings.") + raise EasyBuildError("Invalid value for 'postinstallcmds', should be list or tuple of strings.") for cmd in self.cfg['postinstallcmds']: if not isinstance(cmd, basestring): - self.log.error("Invalid element in 'postinstallcmds', not a string: %s" % cmd) + raise EasyBuildError("Invalid element in 'postinstallcmds', not a string: %s", cmd) run_cmd(cmd, simple=True, log_ok=True, log_all=True) if self.group is not None: @@ -1470,7 +1518,7 @@ def post_install_step(self): adjust_permissions(self.installdir, perms, add=False, recursive=True, group_id=self.group[1], relative=True, ignore_errors=True) except EasyBuildError, err: - self.log.error("Unable to change group permissions of file(s): %s" % err) + raise EasyBuildError("Unable to change group permissions of file(s): %s", err) self.log.info("Successfully made software only available for group %s (gid %s)" % self.group) if read_only_installdir(): @@ -1516,15 +1564,16 @@ def sanity_check_step(self, custom_paths=None, custom_commands=None, extension=F lenvals = [len(x) for x in paths.values()] req_keys = sorted(path_keys_and_check.keys()) if not ks == req_keys or sum(valnottypes) > 0 or sum(lenvals) == 0: - self.log.error("Incorrect format for sanity_check_paths (should have %s keys, " - "values should be lists (at least one non-empty))." % '/'.join(req_keys)) + raise EasyBuildError("Incorrect format for sanity_check_paths (should (only) have %s keys, " + "values should be lists (at least one non-empty)).", ','.join(req_keys)) for key, check_fn in path_keys_and_check.items(): for xs in paths[key]: if isinstance(xs, basestring): xs = (xs,) elif not isinstance(xs, tuple): - self.log.error("Unsupported type '%s' encountered in %s, not a string or tuple" % (key, type(xs))) + raise EasyBuildError("Unsupported type '%s' encountered in %s, not a string or tuple", + key, type(xs)) found = False for name in xs: path = os.path.join(self.installdir, name) @@ -1549,7 +1598,11 @@ def sanity_check_step(self, custom_paths=None, custom_commands=None, extension=F self.log.warning("Sanity check: %s" % self.sanity_check_fail_msgs[-1]) # chdir to installdir (better environment for running tests) - os.chdir(self.installdir) + if os.path.isdir(self.installdir): + try: + os.chdir(self.installdir) + except OSError, err: + raise EasyBuildError("Failed to move to installdir %s: %s", self.installdir, err) # run sanity check commands commands = self.cfg['sanity_check_commands'] @@ -1599,7 +1652,7 @@ def sanity_check_step(self, custom_paths=None, custom_commands=None, extension=F # pass or fail if self.sanity_check_fail_msgs: - self.log.error("Sanity check failed: %s" % ', '.join(self.sanity_check_fail_msgs)) + raise EasyBuildError("Sanity check failed: %s", ', '.join(self.sanity_check_fail_msgs)) else: self.log.debug("Sanity check passed!") @@ -1626,7 +1679,7 @@ def cleanup_step(self): base = os.path.dirname(base) except OSError, err: - self.log.exception("Cleaning up builddir %s failed: %s" % (self.builddir, err)) + raise EasyBuildError("Cleaning up builddir %s failed: %s", self.builddir, err) if not build_option('cleanup_builddir'): self.log.info("Keeping builddir %s" % self.builddir) @@ -1638,19 +1691,20 @@ def make_module_step(self, fake=False): Generate a module file. """ self.module_generator.set_fake(fake) - modpath = self.module_generator.prepare() - txt = '' - txt += self.make_module_description() - txt += self.make_module_extend_modpath() + mod_symlink_paths = ActiveMNS().det_module_symlink_paths(self.cfg) + modpath = self.module_generator.prepare(mod_symlink_paths) + + txt = self.make_module_description() txt += self.make_module_dep() + txt += self.make_module_extend_modpath() txt += self.make_module_req() txt += self.make_module_extra() txt += self.make_module_footer() write_file(self.module_generator.filename, txt) - self.log.info("Module file %s written" % self.module_generator.filename) + self.log.info("Module file %s written: %s", self.module_generator.filename, txt) # only update after generating final module file if not fake: @@ -1677,13 +1731,13 @@ def test_cases_step(self): if os.path.exists(path): break if not os.path.exists(path): - self.log.error("Test specifies invalid path: %s" % path) + raise EasyBuildError("Test specifies invalid path: %s", path) try: self.log.debug("Running test %s" % path) run_cmd(path, log_all=True, simple=True) except EasyBuildError, err: - self.log.exception("Running test %s failed: %s" % (path, err)) + raise EasyBuildError("Running test %s failed: %s", path, err) def update_config_template_run_step(self): """Update the the easyconfig template dictionary with easyconfig.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP names""" @@ -1692,23 +1746,47 @@ def update_config_template_run_step(self): self.cfg.template_values[name[0]] = str(getattr(self, name[0], None)) self.cfg.generate_template_values() - def run_step(self, step, methods, skippable=False): - """ - Run step, returns false when execution should be stopped - """ + def _skip_step(self, step, skippable): + """Dedice whether or not to skip the specified step.""" + module_only = build_option('module_only') + force = build_option('force') + skip = False + + # skip step if specified as individual (skippable) step if skippable and (self.skip or step in self.cfg['skipsteps']): - self.log.info("Skipping %s step" % step) + self.log.info("Skipping %s step (skip: %s, skipsteps: %s)", step, self.skip, self.cfg['skipsteps']) + skip = True + + # skip step when only generating module file + # * still run sanity check without use of force + # * always run ready & prepare step to set up toolchain + deps + elif module_only and not step in MODULE_ONLY_STEPS: + self.log.info("Skipping %s step (only generating module)", step) + skip = True + + # allow skipping sanity check too when only generating module and force is used + elif module_only and step == SANITYCHECK_STEP and force: + self.log.info("Skipping %s step because of forced module-only mode", step) + skip = True + else: - self.log.info("Starting %s step" % step) - # update the config templates - self.update_config_template_run_step() + self.log.debug("Not skipping %s step (skippable: %s, skip: %s, skipsteps: %s, module_only: %s, force: %s", + step, skippable, self.skip, self.cfg['skipsteps'], module_only, force) - for m in methods: - self.log.info("Running method %s part of step %s" % ('_'.join(m.func_code.co_names), step)) - m(self) + return skip + + def run_step(self, step, methods): + """ + Run step, returns false when execution should be stopped + """ + self.log.info("Starting %s step", step) + self.update_config_template_run_step() + for m in methods: + self.log.info("Running method %s part of step %s" % ('_'.join(m.func_code.co_names), step)) + m(self) if self.cfg['stop'] == step: - self.log.info("Stopping after %s step." % step) + self.log.info("Stopping after %s step.", step) raise StopException(step) @staticmethod @@ -1727,14 +1805,14 @@ def get_step(tag, descr, substeps, skippable, initial=True): (True, lambda x: env.reset_changes()), (True, lambda x: x.handle_iterate_opts()), ] - ready_step_spec = lambda initial: get_step('ready', "creating build dir, resetting environment", + ready_step_spec = lambda initial: get_step(READY_STEP, "creating build dir, resetting environment", ready_substeps, False, initial=initial) source_substeps = [ (False, lambda x: x.checksum_step()), (True, lambda x: x.extract_step()), ] - source_step_spec = lambda initial: get_step('source', "unpacking", source_substeps, True, initial=initial) + source_step_spec = lambda initial: get_step(SOURCE_STEP, "unpacking", source_substeps, True, initial=initial) def prepare_step_spec(initial): """Return prepare step specification.""" @@ -1742,7 +1820,7 @@ def prepare_step_spec(initial): substeps = [lambda x: x.prepare_step()] else: substeps = [lambda x: x.guess_start_dir()] - return ('prepare', 'preparing', substeps, False) + return (PREPARE_STEP, 'preparing', substeps, False) install_substeps = [ (False, lambda x: x.stage_install_step()), @@ -1754,14 +1832,14 @@ def prepare_step_spec(initial): # format for step specifications: (stop_name: (description, list of functions, skippable)) # core steps that are part of the iterated loop - patch_step_spec = ('patch', 'patching', [lambda x: x.patch_step()], True) - configure_step_spec = ('configure', 'configuring', [lambda x: x.configure_step()], True) - build_step_spec = ('build', 'building', [lambda x: x.build_step()], True) - test_step_spec = ('test', 'testing', [lambda x: x.test_step()], True) + patch_step_spec = (PATCH_STEP, 'patching', [lambda x: x.patch_step()], True) + configure_step_spec = (CONFIGURE_STEP, 'configuring', [lambda x: x.configure_step()], True) + build_step_spec = (BUILD_STEP, 'building', [lambda x: x.build_step()], True) + test_step_spec = (TEST_STEP, 'testing', [lambda x: x.test_step()], True) # part 1: pre-iteration + first iteration steps_part1 = [ - ('fetch', 'fetching files', [lambda x: x.fetch_step()], False), + (FETCH_STEP, 'fetching files', [lambda x: x.fetch_step()], False), ready_step_spec(True), source_step_spec(True), patch_step_spec, @@ -1786,19 +1864,19 @@ def prepare_step_spec(initial): ] * (iteration_count - 1) # part 3: post-iteration part steps_part3 = [ - ('extensions', 'taking care of extensions', [lambda x: x.extensions_step()], False), - ('package', 'packaging', [lambda x: x.package_step()], True), - ('postproc', 'postprocessing', [lambda x: x.post_install_step()], True), - ('sanitycheck', 'sanity checking', [lambda x: x.sanity_check_step()], False), - ('cleanup', 'cleaning up', [lambda x: x.cleanup_step()], False), - ('module', 'creating module', [lambda x: x.make_module_step()], False), + (EXTENSIONS_STEP, 'taking care of extensions', [lambda x: x.extensions_step()], False), + (PACKAGE_STEP, 'packaging', [lambda x: x.package_step()], True), + (POSTPROC_STEP, 'postprocessing', [lambda x: x.post_install_step()], True), + (SANITYCHECK_STEP, 'sanity checking', [lambda x: x.sanity_check_step()], False), + (CLEANUP_STEP, 'cleaning up', [lambda x: x.cleanup_step()], False), + (MODULE_STEP, 'creating module', [lambda x: x.make_module_step()], False), ] # full list of steps, included iterated steps steps = steps_part1 + steps_part2 + steps_part3 if run_test_cases: - steps.append(('testcases', 'running test cases', [ + steps.append((TESTCASES_STEP, 'running test cases', [ lambda x: x.load_module(), lambda x: x.test_cases_step(), ], False)) @@ -1817,9 +1895,12 @@ def run_all_steps(self, run_test_cases): print_msg("building and installing %s..." % self.full_mod_name, self.log, silent=self.silent) try: - for (stop_name, descr, step_methods, skippable) in steps: - print_msg("%s..." % descr, self.log, silent=self.silent) - self.run_step(stop_name, step_methods, skippable=skippable) + for (step_name, descr, step_methods, skippable) in steps: + if self._skip_step(step_name, skippable): + print_msg("%s [skipped]" % descr, self.log, silent=self.silent) + else: + print_msg("%s..." % descr, self.log, silent=self.silent) + self.run_step(step_name, step_methods) except StopException: pass @@ -1828,11 +1909,11 @@ def run_all_steps(self, run_test_cases): return True -def build_and_install_one(ecdict, orig_environ): +def build_and_install_one(ecdict, init_env): """ Build the software @param ecdict: dictionary contaning parsed easyconfig + metadata - @param orig_environ: original environment (used to reset environment) + @param init_env: original environment (used to reset environment) """ silent = build_option('silent') @@ -1845,7 +1926,7 @@ def build_and_install_one(ecdict, orig_environ): # restore original environment _log.info("Resetting environment") filetools.errors_found_in_log = 0 - restore_env(orig_environ) + restore_env(init_env) cwd = os.getcwd() @@ -1859,8 +1940,8 @@ def build_and_install_one(ecdict, orig_environ): app = app_class(ecdict['ec']) _log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock)) except EasyBuildError, err: - tup = (name, easyblock, err.msg) - print_error("Failed to get application instance for %s (easyblock: %s): %s" % tup, silent=silent) + print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg), + silent=silent) # application settings stop = build_option('stop') @@ -1928,14 +2009,9 @@ def build_and_install_one(ecdict, orig_environ): # cleanup logs app.close_log() - try: - mkdir(new_log_dir, parents=True) - log_fn = os.path.basename(get_log_filename(app.name, app.version)) - application_log = os.path.join(new_log_dir, log_fn) - shutil.move(app.logfile, application_log) - _log.debug("Moved log file %s to %s" % (app.logfile, application_log)) - except (IOError, OSError), err: - print_error("Failed to move log file %s to new log file %s: %s" % (app.logfile, application_log, err)) + log_fn = os.path.basename(get_log_filename(app.name, app.version)) + application_log = os.path.join(new_log_dir, log_fn) + move_logs(app.logfile, application_log) try: newspec = os.path.join(new_log_dir, "%s-%s.eb" % (app.name, det_full_ec_version(app.cfg))) @@ -2039,6 +2115,10 @@ def perform_step(step, obj, method, logfile): apps.append(instance) base_dir = os.getcwd() + + # keep track of environment right before initiating builds + # note: may be different from ORIG_OS_ENVIRON, since EasyBuild may have defined additional env vars itself by now + # e.g. via easyconfig.handle_allowed_system_deps base_env = copy.deepcopy(os.environ) succes = [] @@ -2067,21 +2147,7 @@ def perform_step(step, obj, method, logfile): # close log and move it app.close_log() - try: - # retain old logs - if os.path.exists(applog): - i = 0 - old_applog = "%s.%d" % (applog, i) - while os.path.exists(old_applog): - i += 1 - old_applog = "%s.%d" % (applog, i) - shutil.move(applog, old_applog) - _log.info("Moved existing log file %s to %s" % (applog, old_applog)) - - shutil.move(app.logfile, applog) - _log.info("Log file moved to %s" % applog) - except IOError, err: - print_error("Failed to move log file %s to new log file %s: %s" % (app.logfile, applog, err)) + move_logs(app.logfile, applog) if app not in build_stopped: # gather build stats diff --git a/easybuild/framework/easyconfig/__init__.py b/easybuild/framework/easyconfig/__init__.py index 24e58f0208..906b4eb0d3 100644 --- a/easybuild/framework/easyconfig/__init__.py +++ b/easybuild/framework/easyconfig/__init__.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/constants.py b/easybuild/framework/easyconfig/constants.py index 7183c0c9fb..501db4dcca 100644 --- a/easybuild/framework/easyconfig/constants.py +++ b/easybuild/framework/easyconfig/constants.py @@ -1,5 +1,5 @@ # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,8 +37,12 @@ _log = fancylogger.getLogger('easyconfig.constants', fname=False) + +EXTERNAL_MODULE_MARKER = 'EXTERNAL_MODULE' + # constants that can be used in easyconfig EASYCONFIG_CONSTANTS = { + 'EXTERNAL_MODULE': (EXTERNAL_MODULE_MARKER, "External module marker"), 'SYS_PYTHON_VERSION': (platform.python_version(), "System Python version (platform.python_version())"), 'OS_TYPE': (get_os_type(), "System type (e.g. 'Linux' or 'Darwin')"), 'OS_NAME': (get_os_name(), "System name (e.g. 'fedora' or 'RHEL')"), diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 0d33bd8388..64cf46be16 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -35,6 +35,8 @@ """ from vsc.utils import fancylogger +from easybuild.tools.build_log import EasyBuildError + _log = fancylogger.getLogger('easyconfig.default', fname=False) @@ -108,7 +110,7 @@ 'stop': [None, 'Keyword to halt the build process after a certain step.', BUILD], 'tests': [[], ("List of test-scripts to run after install. A test script should return a " "non-zero exit status to fail"), BUILD], - 'unpack_options': [None, "Extra options for unpacking source", BUILD], + 'unpack_options': ['', "Extra options for unpacking source", BUILD], 'unwanted_env_vars': [[], "List of environment variables that shouldn't be set during build", BUILD], 'versionprefix': ['', ('Additional prefix for software version ' '(placed before version and toolchain name)'), BUILD], @@ -156,6 +158,7 @@ 'modextrapaths': [{}, "Extra paths to be prepended in module file", MODULES], 'modextravars': [{}, "Extra environment variables to be added to module file", MODULES], 'modloadmsg': [{}, "Message that should be printed when generated module is loaded", MODULES], + 'modluafooter': ["", "Footer to include in generated module file (Lua syntax)", MODULES], 'modtclfooter': ["", "Footer to include in generated module file (Tcl syntax)", MODULES], 'modaliases': [{}, "Aliases to be defined in module file", MODULES], 'moduleclass': ['base', 'Module class to be used for this software', MODULES], @@ -180,7 +183,7 @@ def sorted_categories(): def get_easyconfig_parameter_default(param): """Get default value for given easyconfig parameter.""" if param not in DEFAULT_CONFIG: - _log.error("Unkown easyconfig parameter: %s (known: %s)" % (param, sorted(DEFAULT_CONFIG.keys()))) + raise EasyBuildError("Unkown easyconfig parameter: %s (known: %s)", param, sorted(DEFAULT_CONFIG.keys())) else: _log.debug("Returning default value for easyconfig parameter %s: %s" % (param, DEFAULT_CONFIG[param][0])) return DEFAULT_CONFIG[param][0] diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 99a94ddb8c..9da551fcd5 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -56,6 +56,7 @@ from easybuild.tools.toolchain.utilities import get_toolchain from easybuild.tools.utilities import remove_unwanted_chars from easybuild.framework.easyconfig import MANDATORY +from easybuild.framework.easyconfig.constants import EXTERNAL_MODULE_MARKER from easybuild.framework.easyconfig.default import DEFAULT_CONFIG from easybuild.framework.easyconfig.format.convert import Dependency from easybuild.framework.easyconfig.format.one import retrieve_blocks_in_spec @@ -114,7 +115,7 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) if path is not None and not os.path.isfile(path): - self.log.error("EasyConfig __init__ expected a valid path") + raise EasyBuildError("EasyConfig __init__ expected a valid path") # read easyconfig file contents (or use provided rawtxt), so it can be passed down to avoid multiple re-reads self.path = None @@ -168,6 +169,8 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi 'stop': self.valid_stops, } + self.external_modules_metadata = build_option('external_modules_metadata') + # parse easyconfig file self.build_specs = build_specs self.parse() @@ -215,7 +218,7 @@ def update(self, key, value): elif isinstance(prev_value, list): self[key] = prev_value + value else: - self.log.error("Can't update configuration value for %s, because it's not a string or list." % key) + raise EasyBuildError("Can't update configuration value for %s, because it's not a string or list.", key) def parse(self): """ @@ -228,7 +231,8 @@ def parse(self): # build a new dictionary with only the expected keys, to pass as named arguments to get_config_dict() arg_specs = self.build_specs else: - self.log.error("Specifications should be specified using a dictionary, got %s" % type(self.build_specs)) + raise EasyBuildError("Specifications should be specified using a dictionary, got %s", + type(self.build_specs)) self.log.debug("Obtained specs dict %s" % arg_specs) self.log.info("Parsing easyconfig file %s with rawcontent: %s" % (self.path, self.rawtxt)) @@ -241,7 +245,7 @@ def parse(self): # this includes both generic mandatory parameters and software-specific parameters defined via extra_options missing_mandatory_keys = [key for key in self.mandatory if key not in local_vars] if missing_mandatory_keys: - self.log.error("mandatory parameters not provided in %s: %s" % (self.path, missing_mandatory_keys)) + raise EasyBuildError("mandatory parameters not provided in %s: %s", self.path, missing_mandatory_keys) # provide suggestions for typos possible_typos = [(key, difflib.get_close_matches(key.lower(), self._config.keys(), 1, 0.85)) @@ -249,8 +253,8 @@ def parse(self): typos = [(key, guesses[0]) for (key, guesses) in possible_typos if len(guesses) == 1] if typos: - self.log.error("You may have some typos in your easyconfig file: %s" % - ', '.join(["%s -> %s" % typo for typo in typos])) + raise EasyBuildError("You may have some typos in your easyconfig file: %s", + ', '.join(["%s -> %s" % typo for typo in typos])) # we need toolchain to be set when we call _parse_dependency for key in ['toolchain'] + local_vars.keys(): @@ -263,8 +267,7 @@ def parse(self): self[key] = [self._parse_dependency(dep, hidden=True) for dep in local_vars[key]] else: self[key] = local_vars[key] - tup = (key, self[key], type(self[key])) - self.log.info("setting config option %s: value %s (type: %s)" % tup) + self.log.info("setting config option %s: value %s (type: %s)", key, self[key], type(self[key])) elif key in REPLACED_PARAMETERS: _log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0') @@ -280,8 +283,10 @@ def parse(self): def handle_allowed_system_deps(self): """Handle allowed system dependencies.""" for (name, version) in self['allow_system_deps']: - env.setvar(get_software_root_env_var_name(name), name) # root is set to name, not an actual path - env.setvar(get_software_version_env_var_name(name), version) # version is expected to be something that makes sense + # root is set to name, not an actual path + env.setvar(get_software_root_env_var_name(name), name) + # version is expected to be something that makes sense + env.setvar(get_software_version_env_var_name(name), version) def validate(self, check_osdeps=True): """ @@ -302,8 +307,8 @@ def validate(self, check_osdeps=True): self.log.info("Checking skipsteps") if not isinstance(self._config['skipsteps'][0], (list, tuple,)): - self.log.error('Invalid type for skipsteps. Allowed are list or tuple, got %s (%s)' % - (type(self._config['skipsteps'][0]), self._config['skipsteps'][0])) + raise EasyBuildError('Invalid type for skipsteps. Allowed are list or tuple, got %s (%s)', + type(self._config['skipsteps'][0]), self._config['skipsteps'][0]) self.log.info("Checking build option lists") self.validate_iterate_opts_lists() @@ -317,12 +322,12 @@ def validate_license(self): if lic is None: # when mandatory, remove this possibility if 'software_license' in self.mandatory: - self.log.error("License is mandatory, but 'software_license' is undefined") + raise EasyBuildError("License is mandatory, but 'software_license' is undefined") elif not isinstance(lic, License): - self.log.error('License %s has to be a License subclass instance, found classname %s.' % - (lic, lic.__class__.__name__)) + raise EasyBuildError('License %s has to be a License subclass instance, found classname %s.', + lic, lic.__class__.__name__) elif not lic.name in EASYCONFIG_LICENSES_DICT: - self.log.error('Invalid license %s (classname: %s).' % (lic.name, lic.__class__.__name__)) + raise EasyBuildError('Invalid license %s (classname: %s).', lic.name, lic.__class__.__name__) # TODO, when GROUP_SOURCE and/or GROUP_BINARY is True # check the owner of source / binary (must match 'group' parameter from easyconfig) @@ -340,13 +345,14 @@ def validate_os_deps(self): if isinstance(dep, basestring): dep = (dep,) elif not isinstance(dep, tuple): - self.log.error("Non-tuple value type for OS dependency specification: %s (type %s)" % (dep, type(dep))) + raise EasyBuildError("Non-tuple value type for OS dependency specification: %s (type %s)", + dep, type(dep)) if not any([check_os_dependency(cand_dep) for cand_dep in dep]): not_found.append(dep) if not_found: - self.log.error("One or more OS dependencies were not found: %s" % not_found) + raise EasyBuildError("One or more OS dependencies were not found: %s", not_found) else: self.log.info("OS dependencies ok: %s" % self['osdependencies']) @@ -365,7 +371,7 @@ def validate_iterate_opts_lists(self): # anticipate changes in available easyconfig parameters (e.g. makeopts -> buildopts?) if self.get(opt, None) is None: - self.log.error("%s not available in self.cfg (anymore)?!" % opt) + raise EasyBuildError("%s not available in self.cfg (anymore)?!", opt) # keep track of list, supply first element as first option to handle if isinstance(self[opt], (list, tuple)): @@ -374,7 +380,7 @@ def validate_iterate_opts_lists(self): # make sure that options that specify lists have the same length list_opt_lengths = [length for (opt, length) in opt_counts if length > 1] if len(nub(list_opt_lengths)) > 1: - self.log.error("Build option lists for iterated build should have same length: %s" % opt_counts) + raise EasyBuildError("Build option lists for iterated build should have same length: %s", opt_counts) return True @@ -387,10 +393,14 @@ def filter_hidden_deps(self): faulty_deps = [] for hidden_dep in self['hiddendependencies']: # check whether hidden dep is a listed dep using *visible* module name, not hidden one + hidden_mod_name = ActiveMNS().det_full_module_name(hidden_dep) visible_mod_name = ActiveMNS().det_full_module_name(hidden_dep, force_visible=True) if visible_mod_name in dep_mod_names: self['dependencies'] = [d for d in self['dependencies'] if d['full_mod_name'] != visible_mod_name] self.log.debug("Removed dependency matching hidden dependency %s" % hidden_dep) + elif hidden_mod_name in dep_mod_names: + self['dependencies'] = [d for d in self['dependencies'] if d['full_mod_name'] != hidden_mod_name] + self.log.debug("Hidden dependency %s is already marked to be installed as hidden module", hidden_dep) else: # hidden dependencies must also be included in list of dependencies; # this is done to try and make easyconfigs portable w.r.t. site-specific policies with minimal effort, @@ -399,8 +409,8 @@ def filter_hidden_deps(self): faulty_deps.append(visible_mod_name) if faulty_deps: - tup = (faulty_deps, dep_mod_names) - self.log.error("Hidden dependencies with visible module names %s not in list of dependencies: %s" % tup) + raise EasyBuildError("Hidden dependencies with visible module names %s not in list of dependencies: %s", + faulty_deps, dep_mod_names) def dependencies(self): """ @@ -515,7 +525,7 @@ def _validate(self, attr, values): # private method if values is None: values = [] if self[attr] and self[attr] not in values: - self.log.error("%s provided '%s' is not valid: %s" % (attr, self[attr], values)) + raise EasyBuildError("%s provided '%s' is not valid: %s", attr, self[attr], values) # private method def _parse_dependency(self, dep, hidden=False): @@ -527,7 +537,8 @@ def _parse_dependency(self, dep, hidden=False): of these attributes, 'name' and 'version' are mandatory output dict contains these attributes: - ['name', 'version', 'versionsuffix', 'dummy', 'toolchain', 'short_mod_name', 'full_mod_name', 'hidden'] + ['name', 'version', 'versionsuffix', 'dummy', 'toolchain', 'short_mod_name', 'full_mod_name', 'hidden', + 'external_module'] @param hidden: indicate whether corresponding module file should be installed hidden ('.'-prefixed) """ @@ -536,20 +547,31 @@ def _parse_dependency(self, dep, hidden=False): attr = ['name', 'version', 'versionsuffix', 'toolchain'] dependency = { - 'dummy': False, - 'full_mod_name': None, # full module name - 'short_mod_name': None, # short module name - 'name': '', # software name - 'toolchain': None, - 'version': '', + # full/short module names + 'full_mod_name': None, + 'short_mod_name': None, + # software name, version, versionsuffix + 'name': None, + 'version': None, 'versionsuffix': '', + # toolchain with which this dependency is installed + 'toolchain': None, + # boolean indicating whether we're dealing with a dummy toolchain for this dependency + 'dummy': False, + # boolean indicating whether the module for this dependency is (to be) installed hidden 'hidden': hidden, + # boolean indicating whether this dependency should be resolved via an external module + 'external_module': False, + # metadata in case this is an external module; + # provides information on what this module represents (software name/version, install prefix, ...) + 'external_module_metadata': {}, } if isinstance(dep, dict): dependency.update(dep) # make sure 'dummy' key is handled appropriately if 'dummy' in dep and not 'toolchain' in dep: dependency['toolchain'] = dep['dummy'] + elif isinstance(dep, Dependency): dependency['name'] = dep.name() dependency['version'] = dep.version() @@ -559,12 +581,35 @@ def _parse_dependency(self, dep, hidden=False): toolchain = dep.toolchain() if toolchain is not None: dependency['toolchain'] = toolchain + elif isinstance(dep, (list, tuple)): - # try and convert to list - dep = list(dep) - dependency.update(dict(zip(attr, dep))) + if dep and dep[-1] == EXTERNAL_MODULE_MARKER: + if len(dep) == 2: + dependency['external_module'] = True + dependency['short_mod_name'] = dep[0] + dependency['full_mod_name'] = dep[0] + if dep[0] in self.external_modules_metadata: + dependency['external_module_metadata'].update(self.external_modules_metadata[dep[0]]) + self.log.info("Updated dependency info with available metadata for external module %s: %s", + dep[0], dependency['external_module_metadata']) + else: + self.log.info("No metadata available for external module %s", dep[0]) + else: + raise EasyBuildError("Incorrect external dependency specification: %s", dep) + else: + # non-external dependency: tuple (or list) that specifies name/version(/versionsuffix(/toolchain)) + dependency.update(dict(zip(attr, dep))) + else: - self.log.error('Dependency %s of unsupported type: %s.' % (dep, type(dep))) + raise EasyBuildError("Dependency %s of unsupported type: %s", dep, type(dep)) + + if dependency['external_module']: + self.log.debug("Returning parsed external dependency: %s", dependency) + return dependency + + # check whether this dependency should be hidden according to --hide-deps + if build_option('hide_deps'): + dependency['hidden'] |= dependency['name'] in build_option('hide_deps') # dependency inherits toolchain, unless it's specified to have a custom toolchain tc = copy.deepcopy(self['toolchain']) @@ -578,27 +623,29 @@ def _parse_dependency(self, dep, hidden=False): if len(tc_spec) == 2: tc = {'name': tc_spec[0], 'version': tc_spec[1]} else: - self.log.error("List/tuple value for toolchain should have two elements (%s)" % str(tc_spec)) + raise EasyBuildError("List/tuple value for toolchain should have two elements (%s)", str(tc_spec)) elif isinstance(tc_spec, dict): if 'name' in tc_spec and 'version' in tc_spec: tc = copy.deepcopy(tc_spec) else: - self.log.error("Found toolchain spec as dict with required 'name'/'version' keys: %s" % tc_spec) + raise EasyBuildError("Found toolchain spec as dict with wrong keys (no name/version): %s", tc_spec) else: - self.log.error("Unsupported type for toolchain spec encountered: %s => %s" % (tc_spec, type(tc_spec))) + raise EasyBuildError("Unsupported type for toolchain spec encountered: %s (%s)", tc_spec, type(tc_spec)) + self.log.debug("Derived toolchain to use for dependency %s, based on toolchain spec %s: %s", dep, tc_spec, tc) dependency['toolchain'] = tc # make sure 'dummy' value is set correctly dependency['dummy'] = dependency['toolchain']['name'] == DUMMY_TOOLCHAIN_NAME # validations - if not dependency['name']: - self.log.error("Dependency specified without name: %s" % dependency) + if dependency['name'] is None: + raise EasyBuildError("Dependency specified without name: %s", dependency) - if not dependency['version']: - self.log.error("Dependency specified without version: %s" % dependency) + if dependency['version'] is None: + raise EasyBuildError("Dependency specified without version: %s", dependency) + # set module names dependency['short_mod_name'] = ActiveMNS().det_short_module_name(dependency) dependency['full_mod_name'] = ActiveMNS().det_full_module_name(dependency) @@ -643,7 +690,7 @@ def __getitem__(self, key): if key in self._config: value = self._config[key][0] else: - self.log.error("Use of unknown easyconfig parameter '%s' when getting parameter value" % key) + raise EasyBuildError("Use of unknown easyconfig parameter '%s' when getting parameter value", key) if self.enable_templating: if self.template_values is None or len(self.template_values) == 0: @@ -658,8 +705,8 @@ def __setitem__(self, key, value): if key in self._config: self._config[key][0] = value else: - tup = (key, value) - self.log.error("Use of unknown easyconfig parameter '%s' when setting parameter value to '%s'" % tup) + raise EasyBuildError("Use of unknown easyconfig parameter '%s' when setting parameter value to '%s'", + key, value) @handle_deprecated_or_replaced_easyconfig_parameters def get(self, key, default=None): @@ -692,18 +739,6 @@ def det_installversion(version, toolchain_name, toolchain_version, prefix, suffi _log.nosupport('Use det_full_ec_version from easybuild.tools.module_generator instead of %s' % old_fn, '2.0') -def fetch_parameter_from_easyconfig_file(path, param): - """ - Fetch parameter specification from given easyconfig file. - DEPRECATED: use fetch_parameters_from_easyconfig from easybuild.framework.easyconfigs.parser instead - """ - old = 'fetch_parameter_from_easyconfig_file' - new = 'fetch_parameters_from_easyconfig' - _log.deprecated("%s is replaced by %s from easybuild.framework.easyconfig.parser" % (old, new), '3.0') - ectxt = read_file(path) - return fetch_parameters_from_easyconfig(ectxt, [param])[0] - - def get_easyblock_class(easyblock, name=None, default_fallback=True, error_on_failed_import=True): """ Get class for a particular easyblock (or use default) @@ -717,8 +752,8 @@ def get_easyblock_class(easyblock, name=None, default_fallback=True, error_on_fa # figure out if full path was specified or not if es: modulepath = '.'.join(es) - tup = (class_name, modulepath) - _log.info("Assuming that full easyblock module path was specified (class: %s, modulepath: %s)" % tup) + _log.info("Assuming that full easyblock module path was specified (class: %s, modulepath: %s)", + class_name, modulepath) cls = get_class_for(modulepath, class_name) else: # if we only get the class name, most likely we're dealing with a generic easyblock @@ -775,16 +810,17 @@ def get_easyblock_class(easyblock, name=None, default_fallback=True, error_on_fa _log.nosupport(depr_msg, '2.0') else: if error_on_failed_import: - _log.error("Failed to import easyblock for %s because of module issue: %s" % (class_name, err)) + raise EasyBuildError("Failed to import easyblock for %s because of module issue: %s", + class_name, err) else: _log.debug("Failed to import easyblock for %s, but ignoring it: %s" % (class_name, err)) if cls is not None: - tup = (cls.__name__, easyblock, name) - _log.info("Successfully obtained class '%s' for easyblock '%s' (software name '%s')" % tup) + _log.info("Successfully obtained class '%s' for easyblock '%s' (software name '%s')", + cls.__name__, easyblock, name) else: - tup = (easyblock, name, default_fallback) - _log.debug("No class found for easyblock '%s' (software name '%s', default fallback: %s" % tup) + _log.debug("No class found for easyblock '%s' (software name '%s', default fallback: %s", + easyblock, name, default_fallback) return cls @@ -792,7 +828,7 @@ def get_easyblock_class(easyblock, name=None, default_fallback=True, error_on_fa # simply reraise rather than wrapping it into another error raise err except Exception, err: - _log.error("Failed to obtain class for %s easyblock (not available?): %s" % (easyblock, err)) + raise EasyBuildError("Failed to obtain class for %s easyblock (not available?): %s", easyblock, err) def get_module_path(name, generic=False, decode=True): @@ -902,8 +938,7 @@ def process_easyconfig(path, build_specs=None, validate=True, parse_only=False, try: ec = EasyConfig(spec, build_specs=build_specs, validate=validate, hidden=hidden) except EasyBuildError, err: - msg = "Failed to process easyconfig %s:\n%s" % (spec, err.msg) - _log.exception(msg) + raise EasyBuildError("Failed to process easyconfig %s: %s", spec, err.msg) name = ec['name'] @@ -982,7 +1017,7 @@ def robot_find_easyconfig(name, version): return _easyconfig_files_cache[key] paths = build_option('robot_path') if not paths: - _log.error("No robot path specified, which is required when looking for easyconfigs (use --robot)") + raise EasyBuildError("No robot path specified, which is required when looking for easyconfigs (use --robot)") if not isinstance(paths, (list, tuple)): paths = [paths] # candidate easyconfig paths @@ -1014,7 +1049,8 @@ def __init__(self, *args, **kwargs): if sel_mns in avail_mnss: self.mns = avail_mnss[sel_mns]() else: - self.log.error("Selected module naming scheme %s could not be found in %s" % (sel_mns, avail_mnss.keys())) + raise EasyBuildError("Selected module naming scheme %s could not be found in %s", + sel_mns, avail_mnss.keys()) def requires_full_easyconfig(self, keys): """Check whether specified list of easyconfig parameters is sufficient for active module naming scheme.""" @@ -1035,8 +1071,8 @@ def check_ec_type(self, ec): self.log.debug("Full list of parsed easyconfigs: %s" % parsed_ec) ec = parsed_ec[0]['ec'] else: - tup = (ec['name'], det_full_ec_version(ec), ec) - self.log.error("Failed to find easyconfig file '%s-%s.eb' when determining module name for: %s" % tup) + raise EasyBuildError("Failed to find easyconfig file '%s-%s.eb' when determining module name for: %s", + ec['name'], det_full_ec_version(ec), ec) return ec @@ -1059,7 +1095,7 @@ def _det_module_name_with(self, mns_method, ec, force_visible=False): mod_name = mns_method(self.check_ec_type(ec)) if not is_valid_module_name(mod_name): - self.log.error("%s is not a valid module name" % str(mod_name)) + raise EasyBuildError("%s is not a valid module name", str(mod_name)) # check whether module name should be hidden or not # ec may be either a dict or an EasyConfig instance, 'force_visible' argument overrules @@ -1075,6 +1111,13 @@ def det_full_module_name(self, ec, force_visible=False): self.log.debug("Obtained valid full module name %s" % mod_name) return mod_name + def det_install_subdir(self, ec): + """Determine name of software installation subdirectory.""" + self.log.debug("Determining software installation subdir for %s", ec) + subdir = self.mns.det_install_subdir(self.check_ec_type(ec)) + self.log.debug("Obtained subdir %s", subdir) + return subdir + def det_devel_module_filename(self, ec, force_visible=False): """Determine devel module filename.""" modname = self.det_full_module_name(ec, force_visible=force_visible) @@ -1088,9 +1131,8 @@ def det_short_module_name(self, ec, force_visible=False): # sanity check: obtained module name should pass the 'is_short_modname_for' check if not self.is_short_modname_for(mod_name, ec['name']): - tup = (mod_name, ec['name']) - self.log.error("is_short_modname_for('%s', '%s') for active module naming scheme returns False" % tup) - + raise EasyBuildError("is_short_modname_for('%s', '%s') for active module naming scheme returns False", + mod_name, ec['name']) return mod_name def det_module_subdir(self, ec): diff --git a/easybuild/framework/easyconfig/format/__init__.py b/easybuild/framework/easyconfig/format/__init__.py index d9929739b2..0daff8a9cc 100644 --- a/easybuild/framework/easyconfig/format/__init__.py +++ b/easybuild/framework/easyconfig/format/__init__.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/convert.py b/easybuild/framework/easyconfig/format/convert.py index 4f03522275..df3259a871 100644 --- a/easybuild/framework/easyconfig/format/convert.py +++ b/easybuild/framework/easyconfig/format/convert.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2014 Ghent University +# Copyright 2014-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index 7fb06cc2f9..01bc6a4268 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,6 +37,7 @@ from easybuild.framework.easyconfig.format.version import EasyVersion, OrderedVersionOperators from easybuild.framework.easyconfig.format.version import ToolchainVersionOperator, VersionOperator from easybuild.framework.easyconfig.format.convert import Dependency +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.configobj import Section @@ -59,7 +60,7 @@ def get_format_version(txt): maj_min = res.groupdict() format_version = EasyVersion(FORMAT_VERSION_TEMPLATE % maj_min) except (KeyError, TypeError), err: - _log.error("Failed to get version from match %s: %s" % (res.groups(), err)) + raise EasyBuildError("Failed to get version from match %s: %s", res.groups(), err) return format_version @@ -242,13 +243,13 @@ def parse_sections(self, toparse, current): new_value = [] for dep_name, dep_val in value.items(): if isinstance(dep_val, Section): - self.log.error("Unsupported nested section '%s' found in dependencies section" % dep_name) + raise EasyBuildError("Unsupported nested section '%s' in dependencies section", dep_name) else: # FIXME: parse the dependency specification for version, toolchain, suffix, etc. dep = Dependency(dep_val, name=dep_name) if dep.name() is None or dep.version() is None: - tmpl = "Failed to find name/version in parsed dependency: %s (dict: %s)" - self.log.error(tmpl % (dep, dict(dep))) + raise EasyBuildError("Failed to find name/version in parsed dependency: %s (dict: %s)", + dep, dict(dep)) new_value.append(dep) tmpl = 'Converted dependency section %s to %s, passed it to parent section (or default)' @@ -268,7 +269,7 @@ def parse_sections(self, toparse, current): else: self.log.debug("Not a %s section marker" % marker_type.__name__) if not new_key: - self.log.error("Unsupported section marker '%s'" % key) + raise EasyBuildError("Unsupported section marker '%s'", key) # parse value as a section, recursively new_value = self.parse_sections(value, current.get_nested_dict()) @@ -296,10 +297,10 @@ def parse_sections(self, toparse, current): # remove possible surrounding whitespace (some people add space after comma) new_value = [value_type(x.strip()) for x in value] if False in [x.is_valid() for x in new_value]: - self.log.error("Failed to parse '%s' as list of %s" % (value, value_type.__name__)) + raise EasyBuildError("Failed to parse '%s' as list of %s", value, value_type.__name__) else: - tup = (key, value, type(value)) - self.log.error('Bug: supported but unknown key %s with non-string value: %s, type %s' % tup) + raise EasyBuildError('Bug: supported but unknown key %s with non-string value: %s, type %s', + key, value, type(value)) self.log.debug("Converted value '%s' for key '%s' into new value '%s'" % (value, key, new_value)) current[key] = new_value @@ -334,7 +335,7 @@ def parse(self, configobj): self.supported = self.sections.pop(self.SECTION_MARKER_SUPPORTED) for key, value in self.supported.items(): if not key in self.VERSION_OPERATOR_VALUE_TYPES: - self.log.error('Unsupported key %s in %s section' % (key, self.SECTION_MARKER_SUPPORTED)) + raise EasyBuildError('Unsupported key %s in %s section', key, self.SECTION_MARKER_SUPPORTED) self.sections['%s' % key] = value for key, supported_key, fn_name in [('version', 'versions', 'get_version_str'), @@ -344,7 +345,7 @@ def parse(self, configobj): first = self.supported[supported_key][0] f_val = getattr(first, fn_name)() if f_val is None: - self.log.error("First %s %s can't be used as default (%s returned None)" % (key, first, fn_name)) + raise EasyBuildError("First %s %s can't be used as default (%s returned None)", key, first, fn_name) else: self.log.debug('Using first %s (%s) as default %s' % (key, first, f_val)) self.default[key] = f_val @@ -438,8 +439,7 @@ def _squash_netsed_dict(self, key, nested_dict, squashed, sanity, vt_tuple): tc_overops.add(key) if key.test(tcname, tcversion): - tup = (tcname, tcversion, key) - self.log.debug("Found matching marker for specified toolchain '%s, %s': %s" % tup) + self.log.debug("Found matching marker for specified toolchain '%s, %s': %s", tcname, tcversion, key) # TODO remove when unifying add_toolchina with .add() tmp_squashed = self._squash(vt_tuple, nested_dict, sanity) res_sections.update(tmp_squashed.result) @@ -456,7 +456,7 @@ def _squash_netsed_dict(self, key, nested_dict, squashed, sanity, vt_tuple): else: self.log.debug('Found non-matching version marker %s. Ignoring this (nested) section.' % key) else: - self.log.error("Unhandled section marker '%s' (type '%s')" % (key, type(key))) + raise EasyBuildError("Unhandled section marker '%s' (type '%s')", key, type(key)) return res_sections @@ -479,8 +479,8 @@ def _squash_versop(self, key, value, squashed, sanity, vt_tuple): tmp_tc_oversops = {} # temporary, only for conflict checking for tcversop in value: tc_overops = tmp_tc_oversops.setdefault(tcversop.tc_name, OrderedVersionOperators()) - tup = (tcversop, tc_overops, tcname, tcversion) - self.log.debug('Add tcversop %s to tc_overops %s tcname %s tcversion %s' % tup) + self.log.debug("Add tcversop %s to tc_overops %s tcname %s tcversion %s", + tcversop, tc_overops, tcname, tcversion) tc_overops.add(tcversop) # test non-conflicting list if tcversop.test(tcname, tcversion): matching_toolchains.append(tcversop) @@ -507,7 +507,7 @@ def _squash_versop(self, key, value, squashed, sanity, vt_tuple): self.log.debug('No matching versions, removing the whole current key %s' % key) return Squashed() else: - self.log.error('Unexpected VERSION_OPERATOR_VALUE_TYPES key %s value %s' % (key, value)) + raise EasyBuildError('Unexpected VERSION_OPERATOR_VALUE_TYPES key %s value %s', key, value) return None @@ -520,11 +520,11 @@ def get_version_toolchain(self, version=None, tcname=None, tcversion=None): version = self.default['version'] self.log.debug("No version specified, using default %s" % version) else: - self.log.error("No version specified, no default found.") + raise EasyBuildError("No version specified, no default found.") elif version in versions: self.log.debug("Version '%s' is supported in easyconfig." % version) else: - self.log.error("Version '%s' not supported in easyconfig (only %s)" % (version, versions)) + raise EasyBuildError("Version '%s' not supported in easyconfig (only %s)", version, versions) tcnames = [tc.tc_name for tc in self.supported['toolchains']] if tcname is None: @@ -532,11 +532,11 @@ def get_version_toolchain(self, version=None, tcname=None, tcversion=None): tcname = self.default['toolchain']['name'] self.log.debug("No toolchain name specified, using default %s" % tcname) else: - self.log.error("No toolchain name specified, no default found.") + raise EasyBuildError("No toolchain name specified, no default found.") elif tcname in tcnames: self.log.debug("Toolchain '%s' is supported in easyconfig." % tcname) else: - self.log.error("Toolchain '%s' not supported in easyconfig (only %s)" % (tcname, tcnames)) + raise EasyBuildError("Toolchain '%s' not supported in easyconfig (only %s)", tcname, tcnames) tcs = [tc for tc in self.supported['toolchains'] if tc.tc_name == tcname] if tcversion is None: @@ -544,17 +544,16 @@ def get_version_toolchain(self, version=None, tcname=None, tcversion=None): tcversion = self.default['toolchain']['version'] self.log.debug("No toolchain version specified, using default %s" % tcversion) else: - self.log.error("No toolchain version specified, no default found.") + raise EasyBuildError("No toolchain version specified, no default found.") elif any([tc.test(tcname, tcversion) for tc in tcs]): self.log.debug("Toolchain '%s' version '%s' is supported in easyconfig" % (tcname, tcversion)) else: - tup = (tcname, tcversion, tcs) - self.log.error("Toolchain '%s' version '%s' not supported in easyconfig (only %s)" % tup) + raise EasyBuildError("Toolchain '%s' version '%s' not supported in easyconfig (only %s)", + tcname, tcversion, tcs) - tup = (version, tcname, tcversion) - self.log.debug('version %s, tcversion %s, tcname %s' % tup) + self.log.debug('version %s, tcversion %s, tcname %s', version, tcname, tcversion) - return tup + return (version, tcname, tcversion) def get_specs_for(self, version=None, tcname=None, tcversion=None): """ @@ -578,7 +577,7 @@ def __init__(self): self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) if not len(self.VERSION) == len(FORMAT_VERSION_TEMPLATE.split('.')): - self.log.error('Invalid version number %s (incorrect length)' % self.VERSION) + raise EasyBuildError('Invalid version number %s (incorrect length)', self.VERSION) self.rawtext = None # text version of the easyconfig self.header = None # easyconfig header (e.g., format version, license, ...) diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index a4cc036826..da8277b175 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -39,7 +39,7 @@ from easybuild.framework.easyconfig.format.format import FORMAT_DEFAULT_VERSION, get_format_version from easybuild.framework.easyconfig.format.pyheaderconfigobj import EasyConfigFormatConfigObj from easybuild.framework.easyconfig.format.version import EasyVersion -from easybuild.tools.build_log import print_msg +from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.filetools import write_file @@ -71,14 +71,14 @@ def get_config_dict(self): spec_tc_version = spec_tc.get('version', None) cfg = self.pyheader_localvars if spec_version is not None and not spec_version == cfg['version']: - self.log.error('Requested version %s not available, only %s' % (spec_version, cfg['version'])) + raise EasyBuildError('Requested version %s not available, only %s', spec_version, cfg['version']) tc_name = cfg['toolchain']['name'] tc_version = cfg['toolchain']['version'] if spec_tc_name is not None and not spec_tc_name == tc_name: - self.log.error('Requested toolchain name %s not available, only %s' % (spec_tc_name, tc_name)) + raise EasyBuildError('Requested toolchain name %s not available, only %s', spec_tc_name, tc_name) if spec_tc_version is not None and not spec_tc_version == tc_version: - self.log.error('Requested toolchain version %s not available, only %s' % (spec_tc_version, tc_version)) + raise EasyBuildError('Requested toolchain version %s not available, only %s', spec_tc_version, tc_version) return cfg @@ -102,7 +102,7 @@ def retrieve_blocks_in_spec(spec, only_blocks, silent=False): try: txt = open(spec).read() except IOError, err: - _log.error("Failed to read file %s: %s" % (spec, err)) + raise EasyBuildError("Failed to read file %s: %s", spec, err) # split into blocks using regex pieces = reg_block.split(txt) @@ -124,8 +124,7 @@ def retrieve_blocks_in_spec(spec, only_blocks, silent=False): block_contents = pieces.pop(0) if block_name in [b['name'] for b in blocks]: - msg = "Found block %s twice in %s." % (block_name, spec) - _log.error(msg) + raise EasyBuildError("Found block %s twice in %s.", block_name, spec) block = {'name': block_name, 'contents': block_contents} @@ -157,7 +156,7 @@ def retrieve_blocks_in_spec(spec, only_blocks, silent=False): if 'dependencies' in block: for dep in block['dependencies']: if not dep in [b['name'] for b in blocks]: - _log.error("Block %s depends on %s, but block was not found." % (name, dep)) + raise EasyBuildError("Block %s depends on %s, but block was not found.", name, dep) dep = [b for b in blocks if b['name'] == dep][0] txt += "\n# Dependency block %s" % (dep['name']) diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py index 026392824a..78d80be3e4 100644 --- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py +++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -36,6 +36,7 @@ from easybuild.framework.easyconfig.format.format import get_format_version, EasyConfigFormat from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.configobj import ConfigObj from easybuild.tools.systemtools import get_shared_lib_ext @@ -68,7 +69,7 @@ def build_easyconfig_constants_dict(): const_dict[cst_key] = cst_val if len(err) > 0: - _log.error("EasyConfig constants sanity check failed: %s" % ("\n".join(err))) + raise EasyBuildError("EasyConfig constants sanity check failed: %s", '\n'.join(err)) else: return const_dict @@ -128,8 +129,8 @@ def parse(self, txt, strict_section_markers=False): last_n = 100 pre_section_tail = txt[start_section-last_n:start_section] sections_head = txt[start_section:start_section+last_n] - tup = (start_section, last_n, pre_section_tail, sections_head) - self.log.debug('Sections start at index %s, %d-chars context:\n"""%s""""\n\n"""%s..."""' % tup) + self.log.debug('Sections start at index %s, %d-chars context:\n"""%s""""\n\n"""%s..."""', + start_section, last_n, pre_section_tail, sections_head) self.parse_pre_section(txt[:start_section]) if start_section is not None: @@ -148,7 +149,7 @@ def parse_pre_section(self, txt): format_version = get_format_version(line) if format_version is not None: if not format_version == self.VERSION: - self.log.error("Invalid format version %s for current format class" % format_version) + raise EasyBuildError("Invalid format version %s for current format class", format_version) else: self.log.info("Valid format version %s found" % format_version) # version is not part of header @@ -185,7 +186,7 @@ def parse_pyheader(self, pyheader): try: exec(pyheader, global_vars, local_vars) except SyntaxError, err: - self.log.error("SyntaxError in easyconfig pyheader %s: %s" % (pyheader, err)) + raise EasyBuildError("SyntaxError in easyconfig pyheader %s: %s", pyheader, err) self.log.debug("pyheader final global_vars %s" % global_vars) self.log.debug("pyheader final local_vars %s" % local_vars) @@ -230,14 +231,14 @@ def _validate_pyheader(self): blacklisted parameters are not allowed, mandatory parameters are mandatory unless blacklisted """ if self.pyheader_localvars is None: - self.log.error("self.pyheader_localvars must be initialized") + raise EasyBuildError("self.pyheader_localvars must be initialized") if self.PYHEADER_BLACKLIST is None or self.PYHEADER_MANDATORY is None: - self.log.error('Both PYHEADER_BLACKLIST and PYHEADER_MANDATORY must be set') + raise EasyBuildError('Both PYHEADER_BLACKLIST and PYHEADER_MANDATORY must be set') for param in self.PYHEADER_BLACKLIST: if param in self.pyheader_localvars: # TODO add to easyconfig unittest (similar to mandatory) - self.log.error('blacklisted param %s not allowed in pyheader' % param) + raise EasyBuildError('blacklisted param %s not allowed in pyheader', param) missing = [] for param in self.PYHEADER_MANDATORY: @@ -246,13 +247,13 @@ def _validate_pyheader(self): if not param in self.pyheader_localvars: missing.append(param) if missing: - self.log.error('mandatory parameters not provided in pyheader: %s' % ', '.join(missing)) + raise EasyBuildError('mandatory parameters not provided in pyheader: %s', ', '.join(missing)) def parse_section_block(self, section): """Parse the section block by trying to convert it into a ConfigObj instance""" try: self.configobj = ConfigObj(section.split('\n')) except SyntaxError, err: - self.log.error('Failed to convert section text %s: %s' % (section, err)) + raise EasyBuildError('Failed to convert section text %s: %s', section, err) self.log.debug("Found ConfigObj instance %s" % self.configobj) diff --git a/easybuild/framework/easyconfig/format/two.py b/easybuild/framework/easyconfig/format/two.py index ea48707c99..2c6618cb85 100644 --- a/easybuild/framework/easyconfig/format/two.py +++ b/easybuild/framework/easyconfig/format/two.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,6 +37,7 @@ from easybuild.framework.easyconfig.format.pyheaderconfigobj import EasyConfigFormatConfigObj from easybuild.framework.easyconfig.format.format import EBConfigObj from easybuild.framework.easyconfig.format.version import EasyVersion, ToolchainVersionOperator, VersionOperator +from easybuild.tools.build_log import EasyBuildError class FormatTwoZero(EasyConfigFormatConfigObj): @@ -89,10 +90,10 @@ def _check_docstring(self): maintainers.append(res['name']) if self.AUTHOR_REQUIRED and not authors: - self.log.error("No author in docstring (regex: '%s')" % self.AUTHOR_DOCSTRING_REGEX.pattern) + raise EasyBuildError("No author in docstring (regex: '%s')", self.AUTHOR_DOCSTRING_REGEX.pattern) if self.MAINTAINER_REQUIRED and not maintainers: - self.log.error("No maintainer in docstring (regex: '%s')" % self.MAINTAINER_DOCSTRING_REGEX.pattern) + raise EasyBuildError("No maintainer in docstring (regex: '%s')", self.MAINTAINER_DOCSTRING_REGEX.pattern) def get_config_dict(self): """Return the best matching easyconfig dict""" diff --git a/easybuild/framework/easyconfig/format/version.py b/easybuild/framework/easyconfig/format/version.py index cb9efe28ce..e4252ce460 100644 --- a/easybuild/framework/easyconfig/format/version.py +++ b/easybuild/framework/easyconfig/format/version.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,6 +34,7 @@ from distutils.version import LooseVersion from vsc.utils import fancylogger +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.utilities import search_toolchain @@ -80,7 +81,7 @@ def __init__(self, versop_str=None, error_on_parse_failure=False): """ Initialise VersionOperator instance. @param versop_str: intialise with version operator string - @param error_on_parse_failure: log.error in case of parse error + @param error_on_parse_failure: raise EasyBuildError in case of parse error """ self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) @@ -100,7 +101,7 @@ def parse_error(self, msg): """Special function to deal with parse errors""" # TODO major issue what to do in case of misparse. error or not? if self.error_on_parse_failure: - self.log.error(msg) + raise EasyBuildError(msg) else: self.log.debug(msg) @@ -122,7 +123,7 @@ def set(self, versop_str): """ versop_dict = self.parse_versop_str(versop_str) if versop_dict is None: - self.log.error("Failed to parse '%s' as a version operator string" % versop_str) + raise EasyBuildError("Failed to parse '%s' as a version operator string", versop_str) else: for k, v in versop_dict.items(): setattr(self, k, v) @@ -136,16 +137,16 @@ def test(self, test_version): """ # checks whether this VersionOperator instance is valid using __bool__ function if not self: - self.log.error('Not a valid %s. Not initialised yet?' % self.__class__.__name__) + raise EasyBuildError('Not a valid %s. Not initialised yet?', self.__class__.__name__) if isinstance(test_version, basestring): test_version = self._convert(test_version) elif not isinstance(test_version, EasyVersion): - self.log.error("test: argument should be a basestring or EasyVersion (type %s)" % (type(test_version))) + raise EasyBuildError("test: argument should be a basestring or EasyVersion (type %s)", type(test_version)) res = self.operator(test_version, self.version) - tup = (test_version, self.REVERSE_OPERATOR_MAP[self.operator], self.version, res) - self.log.debug("result of testing expression '%s %s %s': %s" % tup) + self.log.debug("result of testing expression '%s %s %s': %s", + test_version, self.REVERSE_OPERATOR_MAP[self.operator], self.version, res) return res @@ -178,7 +179,7 @@ def __eq__(self, versop): if versop is None: return False elif not isinstance(versop, self.__class__): - self.log.error("Types don't match in comparison: %s, expected %s" % (type(versop), self.__class__)) + raise EasyBuildError("Types don't match in comparison: %s, expected %s", type(versop), self.__class__) return self.version == versop.version and self.operator == versop.operator and self.suffix == versop.suffix def __ne__(self, versop): @@ -270,7 +271,7 @@ def parse_versop_str(self, versop_str, versop_dict=None): versop_dict['versop_str'] = versop_str if not 'versop_str' in versop_dict: - self.log.error('Missing versop_str in versop_dict %s' % versop_dict) + raise EasyBuildError('Missing versop_str in versop_dict %s', versop_dict) version = self._convert(versop_dict['version_str']) operator = self._convert_operator(versop_dict['operator_str'], version=version) @@ -311,8 +312,8 @@ def test_overlap_and_conflict(self, versop_other): versop_msg = "this versop %s and versop_other %s" % (self, versop_other) if not isinstance(versop_other, self.__class__): - self.log.error('overlap/conflict check needs instance of self %s (got type %s)' % - (self.__class__.__name__, type(versop_other))) + raise EasyBuildError("overlap/conflict check needs instance of self %s (got type %s)", + self.__class__.__name__, type(versop_other)) if self == versop_other: self.log.debug("%s are equal. Return overlap True, conflict False." % versop_msg) @@ -424,12 +425,12 @@ def _gt_safe(self, version_gt_op, versop_other): Suffix are not considered. """ if len(self.ORDERED_OPERATORS) != len(self.OPERATOR_MAP): - self.log.error('Inconsistency between ORDERED_OPERATORS and OPERATORS (lists are not of same length)') + raise EasyBuildError("Inconsistency between ORDERED_OPERATORS and OPERATORS (lists are not of same length)") # ensure this function is only used for non-conflicting version operators _, conflict = self.test_overlap_and_conflict(versop_other) if conflict: - self.log.error("Conflicting version operator expressions should not be compared with _gt_safe") + raise EasyBuildError("Conflicting version operator expressions should not be compared with _gt_safe") ordered_operators = [self.OPERATOR_MAP[x] for x in self.ORDERED_OPERATORS] if self.version == versop_other.version: @@ -530,7 +531,7 @@ def parse_versop_str(self, tcversop_str): tcversop_dict = super(ToolchainVersionOperator, self).parse_versop_str(None, versop_dict=tcversop_dict) if tcversop_dict.get('version_str', None) is not None and tcversop_dict.get('operator_str', None) is None: - self.log.error("Toolchain version found, but no operator (use ' == '?).") + raise EasyBuildError("Toolchain version found, but no operator (use ' == '?).") self.log.debug("toolchain versop expression '%s' parsed to '%s'" % (tcversop_str, tcversop_dict)) return tcversop_dict @@ -552,15 +553,14 @@ def test(self, name, version): """ # checks whether this ToolchainVersionOperator instance is valid using __bool__ function if not self: - self.log.error('Not a valid %s. Not initialised yet?' % self.__class__.__name__) + raise EasyBuildError('Not a valid %s. Not initialised yet?', self.__class__.__name__) tc_name_res = name == self.tc_name if not tc_name_res: self.log.debug('Toolchain name %s different from test toolchain name %s' % (self.tc_name, name)) version_res = super(ToolchainVersionOperator, self).test(version) res = tc_name_res and version_res - tup = (tc_name_res, version_res, res) - self.log.debug("result of testing expression tc_name_res %s version_res %s: %s" % tup) + self.log.debug("result of testing expression tc_name_res %s version_res %s: %s", tc_name_res, version_res, res) return res @@ -622,8 +622,8 @@ def add(self, versop_new, data=None, update=None): if isinstance(versop_new, basestring): versop_new = VersionOperator(versop_new) elif not isinstance(versop_new, VersionOperator): - tup = (versop_new, type(versop_new)) - self.log.error(("add: argument must be a VersionOperator instance or basestring: %s; type %s") % tup) + raise EasyBuildError("add: argument must be a VersionOperator instance or basestring: %s; type %s", + versop_new, type(versop_new)) if versop_new in self.versops: self.log.debug("Versop %s already added." % versop_new) @@ -632,9 +632,9 @@ def add(self, versop_new, data=None, update=None): gt_test = [versop_new > versop for versop in self.versops] if None in gt_test: # conflict - msg = 'add: conflict(s) between versop_new %s and existing versions %s' conflict_versops = [(idx, self.versops[idx]) for idx, gt_val in enumerate(gt_test) if gt_val is None] - self.log.error(msg % (versop_new, conflict_versops)) + raise EasyBuildError("add: conflict(s) between versop_new %s and existing versions %s", + versop_new, conflict_versops) else: if True in gt_test: # determine first element for which comparison is True @@ -655,8 +655,8 @@ def _add_data(self, versop_new, data, update): if update and versop_new_str in self.datamap: self.log.debug("Keeping track of data for %s UPDATE: %s" % (versop_new_str, data)) if not hasattr(self.datamap[versop_new_str], 'update'): - tup = (versop_new_str, type(self.datamap[versop_new_str])) - self.log.error("Can't update on datamap key %s type %s" % tup) + raise EasyBuildError("Can't update on datamap key %s type %s", + versop_new_str, type(self.datamap[versop_new_str])) self.datamap[versop_new_str].update(data) else: self.log.debug("Keeping track of data for %s SET: %s" % (versop_new_str, data)) @@ -665,11 +665,11 @@ def _add_data(self, versop_new, data, update): def get_data(self, versop): """Return the data for versop from datamap""" if not isinstance(versop, VersionOperator): - tup = (versop, type(versop)) - self.log.error(("get_data: argument must be a VersionOperator instance: %s; type %s") % tup) + raise EasyBuildError("get_data: argument must be a VersionOperator instance: %s; type %s", + versop, type(versop)) versop_str = str(versop) if versop_str in self.datamap: return self.datamap[versop_str] else: - self.log.error('No data in datamap for versop %s' % versop) + raise EasyBuildError("No data in datamap for versop %s", versop) diff --git a/easybuild/framework/easyconfig/licenses.py b/easybuild/framework/easyconfig/licenses.py index ce49c3d5dc..7485cfd549 100644 --- a/easybuild/framework/easyconfig/licenses.py +++ b/easybuild/framework/easyconfig/licenses.py @@ -1,5 +1,5 @@ # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/parser.py b/easybuild/framework/easyconfig/parser.py index 3b4c5ce318..3937ad1833 100644 --- a/easybuild/framework/easyconfig/parser.py +++ b/easybuild/framework/easyconfig/parser.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -35,6 +35,7 @@ from easybuild.framework.easyconfig.format.format import FORMAT_DEFAULT_VERSION from easybuild.framework.easyconfig.format.format import get_format_version, get_format_version_classes +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file, write_file @@ -96,7 +97,7 @@ def __init__(self, filename=None, format_version=None, rawcontent=None): self._check_filename(filename) self.process() else: - self.log.error("Neither filename nor rawcontent provided to EasyConfigParser") + raise EasyBuildError("Neither filename nor rawcontent provided to EasyConfigParser") def process(self, filename=None): """Create an instance""" @@ -112,9 +113,9 @@ def _check_filename(self, fn): self.log.debug("Process filename %s with get function %s, set function %s" % (fn, self.get_fn, self.set_fn)) if self.get_fn is None: - self.log.error('Failed to determine get function for filename %s' % fn) + raise EasyBuildError('Failed to determine get function for filename %s', fn) if self.set_fn is None: - self.log.error('Failed to determine set function for filename %s' % fn) + raise EasyBuildError('Failed to determine set function for filename %s', fn) def _read(self, filename=None): """Read the easyconfig, dump content in self.rawcontent""" @@ -124,11 +125,11 @@ def _read(self, filename=None): try: self.rawcontent = self.get_fn[0](*self.get_fn[1]) except IOError, err: - self.log.error('Failed to obtain content with %s: %s' % (self.get_fn, err)) + raise EasyBuildError('Failed to obtain content with %s: %s', self.get_fn, err) if not isinstance(self.rawcontent, basestring): msg = 'rawcontent is not basestring: type %s, content %s' % (type(self.rawcontent), self.rawcontent) - self.log.error("Unexpected result for raw content: %s" % msg) + raise EasyBuildError("Unexpected result for raw content: %s", msg) def _det_format_version(self): """Extract the format version from the raw content""" @@ -146,10 +147,10 @@ def _get_format_version_class(self): if len(found_classes) == 1: return found_classes[0] elif not found_classes: - self.log.error('No format classes found matching version %s' % self.format_version) + raise EasyBuildError('No format classes found matching version %s', self.format_version) else: - msg = 'More than one format class found matching version %s in %s' % (self.format_version, found_classes) - self.log.error(msg) + raise EasyBuildError("More than one format class found matching version %s in %s", + self.format_version, found_classes) def _set_formatter(self): """Obtain instance of the formatter""" @@ -171,7 +172,7 @@ def write(self, filename=None): try: self.set_fn[0](*self.set_fn[1]) except IOError, err: - self.log.error('Failed to process content with %s: %s' % (self.set_fn, err)) + raise EasyBuildError("Failed to process content with %s: %s", self.set_fn, err) def set_specifications(self, specs): """Set specifications.""" diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 57b973aca2..7608aeece5 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -1,5 +1,5 @@ # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,6 +34,7 @@ from vsc.utils import fancylogger from distutils.version import LooseVersion +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.systemtools import get_shared_lib_ext @@ -87,9 +88,9 @@ 'googlecode.com source url'), ('LAUNCHPAD_SOURCE', 'https://launchpad.net/%(namelower)s/%(version_major_minor)s.x/%(version)s/+download/', 'launchpad.net source url'), - ('PYPI_SOURCE', 'http://pypi.python.org/packages/source/%(nameletter)s/%(name)s', + ('PYPI_SOURCE', 'https://pypi.python.org/packages/source/%(nameletter)s/%(name)s', 'pypi source url'), # e.g., Cython, Sphinx - ('PYPI_LOWER_SOURCE', 'http://pypi.python.org/packages/source/%(nameletterlower)s/%(namelower)s', + ('PYPI_LOWER_SOURCE', 'https://pypi.python.org/packages/source/%(nameletterlower)s/%(namelower)s', 'pypi source url (lowercase name)'), # e.g., Greenlet, PyZMQ ('R_SOURCE', 'http://cran.r-project.org/src/base/R-%(version_major)s', 'cran.r-project.org (base) source url'), @@ -175,7 +176,7 @@ def template_constant_dict(config, ignore=None, skip_lower=True): if softname is not None: template_values['nameletter'] = softname[0] else: - _log.error("Undefined name %s from TEMPLATE_NAMES_EASYCONFIG" % name) + raise EasyBuildError("Undefined name %s from TEMPLATE_NAMES_EASYCONFIG", name) # step 2: add remaining from config for name in TEMPLATE_NAMES_CONFIG: diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 82ad8d0ab0..ed7b118571 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -111,13 +111,24 @@ def find_resolved_modules(unprocessed, avail_modules, retain_all_deps=False): new_ec = ec.copy() deps = [] for dep in new_ec['dependencies']: - full_mod_name = ActiveMNS().det_full_module_name(dep) + full_mod_name = dep.get('full_mod_name', None) + if full_mod_name is None: + full_mod_name = ActiveMNS().det_full_module_name(dep) + dep_resolved = full_mod_name in new_avail_modules if not retain_all_deps: # hidden modules need special care, since they may not be included in list of available modules dep_resolved |= dep['hidden'] and modtool.exist([full_mod_name])[0] + if not dep_resolved: - deps.append(dep) + # treat external modules as resolved when retain_all_deps is enabled (e.g., under --dry-run), + # since no corresponding easyconfig can be found for them + if retain_all_deps and dep.get('external_module', False): + _log.debug("Treating dependency marked as external dependency as resolved: %s", dep) + else: + # no module available (yet) => retain dependency as one to be resolved + deps.append(dep) + new_ec['dependencies'] = deps if len(new_ec['dependencies']) == 0: @@ -144,19 +155,26 @@ def _dep_graph(fn, specs, silent=False): omit_versions = len(names) == len(specs) def mk_node_name(spec): - if omit_versions: - return spec['name'] + if spec.get('external_module', False): + node_name = "%s (EXT)" % spec['full_mod_name'] + elif omit_versions: + node_name = spec['name'] else: - return ActiveMNS().det_full_module_name(spec) + node_name = ActiveMNS().det_full_module_name(spec) + + return node_name # enhance list of specs + all_nodes = set() for spec in specs: spec['module'] = mk_node_name(spec['ec']) + all_nodes.add(spec['module']) spec['unresolved_deps'] = [mk_node_name(s) for s in spec['unresolved_deps']] + all_nodes.update(spec['unresolved_deps']) # build directed graph dgr = digraph() - dgr.add_nodes([spec['module'] for spec in specs]) + dgr.add_nodes(all_nodes) for spec in specs: for dep in spec['unresolved_deps']: dgr.add_edge((spec['module'], dep)) @@ -180,9 +198,8 @@ def dep_graph(*args, **kwargs): try: _dep_graph(*args, **kwargs) except NameError, err: - errors = "\n".join(graph_errors) - msg = "An optional Python packages required to generate dependency graphs is missing: %s" % errors - _log.error("%s\nerr: %s" % (msg, err)) + raise EasyBuildError("An optional Python packages required to generate dependency graphs is missing: %s, %s", + '\n'.join(graph_errors), err) def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None): @@ -240,15 +257,14 @@ def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_pr=False): return tweaked_ecs_path, pr_path -def det_easyconfig_paths(orig_paths, from_pr=None, easyconfigs_pkg_paths=None): +def det_easyconfig_paths(orig_paths): """ Determine paths to easyconfig files. @param orig_paths: list of original easyconfig paths - @param from_pr: pull request number to fetch easyconfigs from - @param easyconfigs_pkg_paths: paths to installed easyconfigs package + @return: list of paths to easyconfig files """ - if easyconfigs_pkg_paths is None: - easyconfigs_pkg_paths = [] + from_pr = build_option('from_pr') + robot_path = build_option('robot_path') # list of specified easyconfig files ec_files = orig_paths[:] @@ -266,8 +282,8 @@ def det_easyconfig_paths(orig_paths, from_pr=None, easyconfigs_pkg_paths=None): # if no easyconfigs are specified, use all the ones touched in the PR ec_files = [path for path in pr_files if path.endswith('.eb')] - if ec_files and easyconfigs_pkg_paths: - # look for easyconfigs with relative paths in easybuild-easyconfigs package, + if ec_files and robot_path: + # look for easyconfigs with relative paths in robot search path, # unless they were found at the given relative paths # determine which easyconfigs files need to be found, if any @@ -277,8 +293,8 @@ def det_easyconfig_paths(orig_paths, from_pr=None, easyconfigs_pkg_paths=None): ecs_to_find.append((idx, ec_file)) _log.debug("List of easyconfig files to find: %s" % ecs_to_find) - # find missing easyconfigs by walking paths with installed easyconfig files - for path in easyconfigs_pkg_paths: + # find missing easyconfigs by walking paths in robot search path + for path in robot_path: _log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path)) for (subpath, dirnames, filenames) in os.walk(path, topdown=True): for idx, orig_path in ecs_to_find[:]: @@ -300,8 +316,7 @@ def det_easyconfig_paths(orig_paths, from_pr=None, easyconfigs_pkg_paths=None): if not ecs_to_find: break - # indicate that specified paths do not contain generated easyconfig files - return [(ec_file, False) for ec_file in ec_files] + return ec_files def parse_easyconfigs(paths): @@ -316,7 +331,7 @@ def parse_easyconfigs(paths): # keep track of whether any files were generated generated_ecs |= generated if not os.path.exists(path): - _log.error("Can't find path %s" % path) + raise EasyBuildError("Can't find path %s", path) try: ec_files = find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs')) for ec_file in ec_files: @@ -327,7 +342,7 @@ def parse_easyconfigs(paths): ecs = process_easyconfig(ec_file, **kwargs) easyconfigs.extend(ecs) except IOError, err: - _log.error("Processing easyconfigs in path %s failed: %s" % (path, err)) + raise EasyBuildError("Processing easyconfigs in path %s failed: %s", path, err) return easyconfigs, generated_ecs @@ -337,7 +352,7 @@ def stats_to_str(stats): Pretty print build statistics to string. """ if not isinstance(stats, (OrderedDict, dict)): - _log.error("Can only pretty print build stats in dictionary form, not of type %s" % type(stats)) + raise EasyBuildError("Can only pretty print build stats in dictionary form, not of type %s", type(stats)) txt = "{\n" pref = " " diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 010a411699..aeca7e2a2e 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -42,7 +42,9 @@ from vsc.utils import fancylogger from vsc.utils.missing import nub +from easybuild.framework.easyconfig.default import get_easyconfig_parameter_default from easybuild.framework.easyconfig.easyconfig import EasyConfig, create_paths, process_easyconfig +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file, write_file from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.robot import resolve_dependencies @@ -74,7 +76,8 @@ def tweak(easyconfigs, build_specs, targetdir=None): # make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble) toolchains = nub(['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs]) if len(toolchains) > 1: - _log.error("Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s" % toolchains) + raise EasyBuildError("Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s", + toolchains) if 'name' in build_specs or 'version' in build_specs: # no recursion if software name/version build specification are included @@ -143,7 +146,7 @@ def tweak_one(src_fn, target_fn, tweaks, targetdir=None): tc_regexp = re.compile(r"^\s*toolchain\s*=\s*(.*)$", re.M) res = tc_regexp.search(ectxt) if not res: - _log.error("No toolchain found in easyconfig file %s?" % src_fn) + raise EasyBuildError("No toolchain found in easyconfig file %s?", src_fn) toolchain = eval(res.group(1)) for key in ['name', 'version']: @@ -162,11 +165,16 @@ def __repr__(self): additions = [] + # automagically clear out list of checksums if software version is being tweaked + if 'version' in tweaks and 'checksums' not in tweaks: + tweaks['checksums'] = [] + _log.warning("Tweaking version: checksums cleared, verification disabled.") + # we need to treat list values seperately, i.e. we prepend to the current value (if any) for (key, val) in tweaks.items(): if isinstance(val, list): - regexp = re.compile(r"^(?P\s*%s)\s*=\s*(?P.*)$" % key, re.M) + regexp = re.compile(r"^(?P\s*%s)\s*=\s*(?P\[(.|\n)*\])\s*$" % key, re.M) res = regexp.search(ectxt) if res: fval = [x for x in val if x != ''] # filter out empty strings @@ -174,7 +182,10 @@ def __repr__(self): # - input ending with comma (empty tail list element) => prepend # - input starting with comma (empty head list element) => append # - no empty head/tail list element => overwrite - if val[0] == '': + if not val: + newval = '[]' + _log.debug("Clearing %s to empty list (was: %s)" % (key, res.group('val'))) + elif val[0] == '': newval = "%s + %s" % (res.group('val'), fval) _log.debug("Appending %s to %s" % (fval, key)) elif val[-1] == '': @@ -185,7 +196,7 @@ def __repr__(self): _log.debug("Overwriting %s with %s" % (key, fval)) ectxt = regexp.sub("%s = %s" % (res.group('key'), newval), ectxt) _log.info("Tweaked %s list to '%s'" % (key, newval)) - else: + elif get_easyconfig_parameter_default(key) != val: additions.append("%s = %s" % (key, val)) tweaks.pop(key) @@ -204,14 +215,14 @@ def __repr__(self): diff = eval(res.group('val')) != val except (NameError, SyntaxError): # if eval fails, just fall back to string comparison - tup = (res.group('val'), val) - _log.debug("eval failed for \"%s\", falling back to string comparison against \"%s\"..." % tup) + _log.debug("eval failed for \"%s\", falling back to string comparison against \"%s\"...", + res.group('val'), val) diff = res.group('val') != val if diff: ectxt = regexp.sub("%s = %s" % (res.group('key'), quote_str(val)), ectxt) _log.info("Tweaked '%s' to '%s'" % (key, quote_str(val))) - else: + elif get_easyconfig_parameter_default(key) != val: additions.append("%s = %s" % (key, quote_str(val))) if additions: @@ -237,7 +248,7 @@ def __repr__(self): # get rid of temporary file os.remove(tmpfn) except OSError, err: - _log.error("Failed to determine suiting filename for tweaked easyconfig file: %s" % err) + raise EasyBuildError("Failed to determine suiting filename for tweaked easyconfig file: %s", err) if targetdir is None: targetdir = tempfile.gettempdir() @@ -265,7 +276,7 @@ def pick_version(req_ver, avail_vers): """ if not avail_vers: - _log.error("Empty list of available versions passed.") + raise EasyBuildError("Empty list of available versions passed.") selected_ver = None if req_ver: @@ -334,7 +345,7 @@ def select_or_generate_ec(fp, paths, specs): # ensure that at least name is specified if not specs.get('name'): - _log.error("Supplied 'specs' dictionary doesn't even contain a name of a software package?") + raise EasyBuildError("Supplied 'specs' dictionary doesn't even contain a name of a software package?") name = specs['name'] handled_params = ['name'] @@ -362,7 +373,8 @@ def select_or_generate_ec(fp, paths, specs): _log.debug("No template found at %s." % templ_file) if len(ec_files) == 0: - _log.error("No easyconfig files found for software %s, and no templates available. I'm all out of ideas." % name) + raise EasyBuildError("No easyconfig files found for software %s, and no templates available. " + "I'm all out of ideas.", name) ecs_and_files = [(EasyConfig(f, validate=False), f) for f in ec_files] @@ -390,8 +402,8 @@ def unique(l): if EASYCONFIG_TEMPLATE in tcnames: _log.info("No easyconfig file for specified toolchain, but template is available.") else: - _log.error("No easyconfig file for %s with toolchain %s, " \ - "and no template available." % (name, specs['toolchain_name'])) + raise EasyBuildError("No easyconfig file for %s with toolchain %s, and no template available.", + name, specs['toolchain_name']) tcname = specs.pop('toolchain_name', None) handled_params.append('toolchain_name') @@ -414,7 +426,7 @@ def unique(l): else: # if multiple toolchains are available, and none is specified, we quit # we can't just pick one, how would we prefer one over the other? - _log.error("No toolchain name specified, and more than one available: %s." % tcnames) + raise EasyBuildError("No toolchain name specified, and more than one available: %s.", tcnames) _log.debug("Filtering easyconfigs based on toolchain name '%s'..." % selected_tcname) ecs_and_files = [x for x in ecs_and_files if x[0]['toolchain']['name'] == selected_tcname] @@ -497,9 +509,7 @@ def unique(l): filter_ecs = True else: # otherwise, we fail, because we don't know how to pick between different fixes - _log.error("No %s specified, and can't pick from available %ses %s" % (param, - param, - vals)) + raise EasyBuildError("No %s specified, and can't pick from available ones: %s", param, vals) if filter_ecs: _log.debug("Filtering easyconfigs based on %s '%s'..." % (param, selected_val)) @@ -515,7 +525,7 @@ def unique(l): cnt = len(ecs_and_files) if not cnt == 1: fs = [x[1] for x in ecs_and_files] - _log.error("Failed to select a single easyconfig from available ones, %s left: %s" % (cnt, fs)) + raise EasyBuildError("Failed to select a single easyconfig from available ones, %s left: %s", cnt, fs) else: (selected_ec, selected_ec_file) = ecs_and_files[0] @@ -570,11 +580,11 @@ def obtain_ec_for(specs, paths, fp=None): # ensure that at least name is specified if not specs.get('name'): - _log.error("Supplied 'specs' dictionary doesn't even contain a name of a software package?") + raise EasyBuildError("Supplied 'specs' dictionary doesn't even contain a name of a software package?") # collect paths to search in if not paths: - _log.error("No paths to look for easyconfig files, specify a path with --robot.") + raise EasyBuildError("No paths to look for easyconfig files, specify a path with --robot.") # select best easyconfig, or try to generate one that fits the requirements res = select_or_generate_ec(fp, paths, specs) @@ -582,4 +592,4 @@ def obtain_ec_for(specs, paths, fp=None): if res: return res else: - _log.error("No easyconfig found for requested software, and also failed to generate one.") + raise EasyBuildError("No easyconfig found for requested software, and also failed to generate one.") diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index a50781e9b3..d8c6f371de 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -36,6 +36,7 @@ import copy import os +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_path from easybuild.tools.run import run_cmd @@ -54,7 +55,7 @@ def __init__(self, mself, ext): self.ext = copy.deepcopy(ext) if not 'name' in self.ext: - self.log.error("'name' is missing in supplied class instance 'ext'.") + raise EasyBuildError("'name' is missing in supplied class instance 'ext'.") self.src = self.ext.get('src', None) self.patches = self.ext.get('patches', None) @@ -111,7 +112,7 @@ def sanity_check_step(self): try: os.chdir(build_path()) except OSError, err: - self.log.error("Failed to change directory: %s" % err) + raise EasyBuildError("Failed to change directory: %s", err) # disabling templating is required here to support legacy string templates like name/version self.cfg.enable_templating = False diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index a7a5ba6488..7f9523f3f0 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of the University of Ghent (http://ugent.be/hpc). @@ -31,6 +31,7 @@ from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extension import Extension +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import apply_patch, extract_file from easybuild.tools.utilities import remove_unwanted_chars @@ -97,7 +98,7 @@ def run(self, unpack_src=False): if self.patches: for patchfile in self.patches: if not apply_patch(patchfile, self.ext_dir): - self.log.error("Applying patch %s failed" % patchfile) + raise EasyBuildError("Applying patch %s failed", patchfile) def sanity_check_step(self, exts_filter=None, custom_paths=None, custom_commands=None): """ @@ -128,7 +129,7 @@ def sanity_check_step(self, exts_filter=None, custom_paths=None, custom_commands if self.is_extension: self.log.warning(msg) else: - self.log.error(msg) + raise EasyBuildError(msg) return False else: self.log.info("Sanity check for %s successful!" % self.name) diff --git a/easybuild/main.py b/easybuild/main.py index f1ab88af41..0c42cd6a2e 100755 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -88,9 +88,9 @@ def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing= _log.warning("Failed to remove generated easyconfig file %s: %s" % (ec_file, err)) # don't use a generated easyconfig unless generation was requested (using a --try-X option) - _log.error(("Unable to find an easyconfig for the given specifications: %s; " - "to make EasyBuild try to generate a matching easyconfig, " - "use the --try-X options ") % build_specs) + raise EasyBuildError("Unable to find an easyconfig for the given specifications: %s; " + "to make EasyBuild try to generate a matching easyconfig, " + "use the --try-X options ", build_specs) return [(ec_file, generated)] @@ -100,13 +100,13 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): # obtain a copy of the starting environment so each build can start afresh # we shouldn't use the environment from init_session_state, since relevant env vars might have been set since # e.g. via easyconfig.handle_allowed_system_deps - orig_environ = copy.deepcopy(os.environ) + init_env = copy.deepcopy(os.environ) res = [] for ec in ecs: ec_res = {} try: - (ec_res['success'], app_log, err) = build_and_install_one(ec, orig_environ) + (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env) ec_res['log_file'] = app_log if not ec_res['success']: ec_res['err'] = EasyBuildError(err) @@ -132,9 +132,9 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): if not ec_res['success'] and exit_on_failure: if 'traceback' in ec_res: - _log.error(ec_res['traceback']) + raise EasyBuildError(ec_res['traceback']) else: - _log.error(test_msg) + raise EasyBuildError(test_msg) res.append((ec, ec_res)) @@ -172,7 +172,8 @@ def main(testing_data=(None, None, None)): # disallow running EasyBuild as root if os.getuid() == 0: - _log.error("You seem to be running EasyBuild with root privileges which is not wise, so let's end this here.") + raise EasyBuildError("You seem to be running EasyBuild with root privileges which is not wise, " + "so let's end this here.") # log startup info eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args @@ -226,8 +227,11 @@ def main(testing_data=(None, None, None)): _log.warning("Failed to determine install path for easybuild-easyconfigs package.") # determine paths to easyconfigs - paths = det_easyconfig_paths(orig_paths, options.from_pr, easyconfigs_pkg_paths) - if not paths: + paths = det_easyconfig_paths(orig_paths) + if paths: + # transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated + paths = [(p, False) for p in paths] + else: if 'name' in build_specs: # try to obtain or generate an easyconfig file via build specifications if a software name is provided paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing) diff --git a/easybuild/scripts/add_header.py b/easybuild/scripts/add_header.py index a7d48f6214..090350e297 100644 --- a/easybuild/scripts/add_header.py +++ b/easybuild/scripts/add_header.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC-UGent team. diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index 9cfb05c6c6..17a6055513 100755 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -46,6 +46,7 @@ import os import re import shutil +import site import sys import tempfile from distutils.version import LooseVersion @@ -56,7 +57,11 @@ EASYBUILD_PACKAGES = [VSC_BASE, 'easybuild-framework', 'easybuild-easyblocks', 'easybuild-easyconfigs'] # set print_debug to True for detailed progress info -print_debug = os.environ.get('EASYBUILD_BOOTSTRAP_DEBUG', False) +print_debug = os.environ.pop('EASYBUILD_BOOTSTRAP_DEBUG', False) + +# don't add user site directory to sys.path (equivalent to python -s), see https://www.python.org/dev/peps/pep-0370/ +os.environ['PYTHONNOUSERSITE'] = '1' +site.ENABLE_USER_SITE = False # clean PYTHONPATH to avoid finding readily installed stuff os.environ['PYTHONPATH'] = '' @@ -362,10 +367,15 @@ def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath): info("\n\n+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n\n" % install_path) - if distribute_egg_dir is not None: - # make sure we still have distribute in PYTHONPATH, so we have control over which 'setup' is used - pythonpaths = [x for x in os.environ.get('PYTHONPATH', '').split(os.pathsep) if len(x) > 0] - os.environ['PYTHONPATH'] = os.pathsep.join([distribute_egg_dir] + pythonpaths) + # inject path to distribute installed in stage 1 into $PYTHONPATH via preinstallopts + # other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH + if distribute_egg_dir is None: + preinstallopts = '' + else: + preinstallopts = 'PYTHONPATH=%s:$PYTHONPATH' % distribute_egg_dir + templates.update({ + 'preinstallopts': preinstallopts, + }) # create easyconfig file ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version']) @@ -373,6 +383,7 @@ def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath): templates.update({ 'source_urls': '\n'.join(["'%s/%s/%s'," % (PYPI_SOURCE_URL, pkg[0], pkg) for pkg in EASYBUILD_PACKAGES]), 'sources': "%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates, + 'pythonpath': distribute_egg_dir, }) f.write(EASYBUILD_EASYCONFIG_TEMPLATE % templates) f.close() @@ -428,11 +439,11 @@ def main(): error("Usage: %s " % sys.argv[0]) install_path = os.path.abspath(sys.argv[1]) - sourcepath = os.environ.get('EASYBUILD_BOOTSTRAP_SOURCEPATH') + sourcepath = os.environ.pop('EASYBUILD_BOOTSTRAP_SOURCEPATH', None) if sourcepath is not None: info("Fetching sources from %s..." % sourcepath) - skip_stage0 = os.environ.get('EASYBUILD_BOOTSTRAP_SKIP_STAGE0', False) + skip_stage0 = os.environ.pop('EASYBUILD_BOOTSTRAP_SKIP_STAGE0', False) # create temporary dir for temporary installations tmpdir = tempfile.mkdtemp() @@ -447,10 +458,13 @@ def main(): sys.path = [] for path in orig_sys_path: include_path = True - # exclude path if it's potentially an EasyBuild package - if 'easybuild' in path: + # exclude path if it's potentially an EasyBuild/VSC package, providing the 'easybuild'/'vsc' namespace, resp. + if any([os.path.exists(os.path.join(path, pkg, '__init__.py')) for pkg in ['easyblocks', 'easybuild', 'vsc']]): include_path = False - # exclude path if it contain an easy-install.pth file + # exclude any .egg paths + if path.endswith('.egg'): + include_path = False + # exclude any path that contains an easy-install.pth file if os.path.exists(os.path.join(path, 'easy-install.pth')): include_path = False @@ -519,6 +533,8 @@ def main(): # EasyBuild is a (set of) Python packages, so it depends on Python # usually, we want to use the system Python, so no actual Python dependency is listed allow_system_deps = [('Python', SYS_PYTHON_VERSION)] + +preinstallopts = '%(preinstallopts)s' """ # distribute_setup.py script (https://pypi.python.org/pypi/distribute) diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py index 14e26a6a14..ff903b4265 100755 --- a/easybuild/scripts/clean_gists.py +++ b/easybuild/scripts/clean_gists.py @@ -31,6 +31,7 @@ from vsc.utils import fancylogger from vsc.utils.generaloption import simple_option from vsc.utils.rest import RestClient +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.github import GITHUB_API_URL, HTTP_STATUS_OK, GITHUB_EASYCONFIGS_REPO from easybuild.tools.github import GITHUB_EB_MAIN, fetch_github_token from easybuild.tools.options import EasyBuildOptions @@ -54,7 +55,7 @@ def main(): log = go.log if not (go.options.all or go.options.closed_pr or go.options.orphans): - log.error("Please tell me what to do?") + raise EasyBuildError("Please tell me what to do?") if go.options.github_user is None: eb_go = EasyBuildOptions(envvar_prefix='EASYBUILD', go_args=[]) @@ -64,7 +65,7 @@ def main(): username = go.options.github_user if username is None: - log.error("Could not find a github username") + raise EasyBuildError("Could not find a github username") else: log.info("Using username = %s", username) @@ -75,8 +76,8 @@ def main(): status, gists = gh.gists.get(per_page=100) if status != HTTP_STATUS_OK: - log.error("Failed to get a lists of gists for user %s: error code %s, message = %s", - username, status, gists) + raise EasyBuildError("Failed to get a lists of gists for user %s: error code %s, message = %s", + username, status, gists) else: log.info("Found %s gists", len(gists)) @@ -102,8 +103,8 @@ def main(): if pr_num not in pr_cache: status, pr = gh.repos[GITHUB_EB_MAIN][GITHUB_EASYCONFIGS_REPO].pulls[pr_num].get() if status != HTTP_STATUS_OK: - log.error("Failed to get pull-request #%s: error code %s, message = %s", - pr_num, status, pr) + raise EasyBuildError("Failed to get pull-request #%s: error code %s, message = %s", + pr_num, status, pr) pr_cache[pr_num] = pr["state"] if pr_cache[pr_num] == "closed": @@ -118,8 +119,8 @@ def main(): status, del_gist = gh.gists[gist["id"]].delete() if status != HTTP_DELETE_OK: - log.error("Unable to remove gist (id=%s): error code %s, message = %s", - gist["id"], status, del_gist) + raise EasyBuildError("Unable to remove gist (id=%s): error code %s, message = %s", + gist["id"], status, del_gist) else: log.info("Delete gist with id=%s", gist["id"]) num_deleted += 1 diff --git a/easybuild/scripts/fix_broken_easyconfigs.py b/easybuild/scripts/fix_broken_easyconfigs.py index 2144eefbb6..7f1cc626a0 100755 --- a/easybuild/scripts/fix_broken_easyconfigs.py +++ b/easybuild/scripts/fix_broken_easyconfigs.py @@ -30,23 +30,19 @@ import os import re import sys -import tempfile from vsc.utils import fancylogger -from vsc.utils.generaloption import simple_option +from vsc.utils.generaloption import SimpleOption -from easybuild.tools.build_log import EasyBuildError from easybuild.framework.easyconfig.easyconfig import get_easyblock_class from easybuild.framework.easyconfig.parser import REPLACED_PARAMETERS, fetch_parameters_from_easyconfig +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import init_build_options from easybuild.tools.filetools import find_easyconfigs, read_file, write_file -from test.framework.utilities import init_config - -options = { - 'backup': ("Backup up easyconfigs before modifying them", None, 'store_true', True, 'b'), -} -go = simple_option(options) -log = go.log +class FixBrokenEasyconfigsOption(SimpleOption): + """Custom option parser for this script.""" + ALLOPTSMANDATORY = False def fix_broken_easyconfig(ectxt, easyblock_class): @@ -101,7 +97,7 @@ def process_easyconfig_file(ec_file): os.rename(ec_file, backup_ec_file) log.info("Backed up %s to %s" % (ec_file, backup_ec_file)) except OSError, err: - log.error("Failed to backup %s before rewriting it: %s" % (ec_file, err)) + raise EasyBuildError("Failed to backup %s before rewriting it: %s", ec_file, err) write_file(ec_file, fixed_ectxt) log.debug("Contents of fixed easyconfig file: %s" % fixed_ectxt) @@ -112,34 +108,38 @@ def process_easyconfig_file(ec_file): # MAIN -def main(): - """Main script functionality.""" +try: + init_build_options() + + options = { + 'backup': ("Backup up easyconfigs before modifying them", None, 'store_true', True, 'b'), + } + go = FixBrokenEasyconfigsOption(options) + log = go.log fancylogger.logToScreen(enable=True, stdout=True) - log.setLevel('INFO') + fancylogger.setLogLevel('WARNING') try: import easybuild.easyblocks.generic.configuremake except ImportError, err: - log.error("easyblocks are not available in Python search path: %s" % err) - - init_config(args=['--quiet']) + raise EasyBuildError("easyblocks are not available in Python search path: %s", err) for path in go.args: if not os.path.exists(path): - log.error("Non-existing path %s specified" % path) + raise EasyBuildError("Non-existing path %s specified", path) ec_files = [ec for p in go.args for ec in find_easyconfigs(p)] if not ec_files: - log.error("No easyconfig files specified") + raise EasyBuildError("No easyconfig files specified") log.info("Processing %d easyconfigs" % len(ec_files)) for ec_file in ec_files: - process_easyconfig_file(ec_file) - - -if __name__ == '__main__': - try: - main() - except EasyBuildError, err: - sys.exit(1) + try: + process_easyconfig_file(ec_file) + except EasyBuildError, err: + log.warning("Ignoring issue when processing %s: %s", ec_file, err) + +except EasyBuildError, err: + sys.stderr.write("ERROR: %s\n" % err) + sys.exit(1) diff --git a/easybuild/scripts/generate_software_list.py b/easybuild/scripts/generate_software_list.py index 94018d05d6..26feaab3b2 100644 --- a/easybuild/scripts/generate_software_list.py +++ b/easybuild/scripts/generate_software_list.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of the University of Ghent (http://ugent.be/hpc). @@ -33,10 +33,10 @@ from datetime import date from optparse import OptionParser -import easybuild.tools.build_log # ensure use of EasyBuildLog import easybuild.tools.config as config import easybuild.tools.options as eboptions from easybuild.framework.easyconfig.easyconfig import EasyConfig, get_easyblock_class +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.github import Githubfs from vsc.utils import fancylogger @@ -101,7 +101,7 @@ # configure EasyBuild, by parsing options eb_go = eboptions.parse_options(args=args) config.init(eb_go.options, eb_go.get_options_by_section('config')) -config.init_build_options({'validate': False}) +config.init_build_options({'validate': False, 'external_modules_metadata': {}}) configs = [] @@ -125,7 +125,7 @@ ec = EasyConfig(ec_file) log.info("found valid easyconfig %s" % ec) if not ec.name in names: - log.info("found new software package %s" % ec) + log.info("found new software package %s" % ec.name) ec.easyblock = None # check if an easyblock exists ebclass = get_easyblock_class(None, name=ec.name, default_fallback=False) @@ -136,7 +136,7 @@ configs.append(ec) names.append(ec.name) except Exception, err: - log.error("faulty easyconfig %s: %s" % (ec_file, err)) + raise EasyBuildError("faulty easyconfig %s: %s", ec_file, err) log.info("Found easyconfigs: %s" % [x.name for x in configs]) # sort by name diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py index 0d57f13f35..e3b1b3e220 100755 --- a/easybuild/scripts/mk_tmpl_easyblock_for.py +++ b/easybuild/scripts/mk_tmpl_easyblock_for.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/port_easyblock.py b/easybuild/scripts/port_easyblock.py index 7fbe90a2d9..515e29edcc 100644 --- a/easybuild/scripts/port_easyblock.py +++ b/easybuild/scripts/port_easyblock.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/prep_for_release.py b/easybuild/scripts/prep_for_release.py index ffc4045807..60b7651e60 100644 --- a/easybuild/scripts/prep_for_release.py +++ b/easybuild/scripts/prep_for_release.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/repo_setup.py b/easybuild/scripts/repo_setup.py index 23686cdd73..2cf4686821 100644 --- a/easybuild/scripts/repo_setup.py +++ b/easybuild/scripts/repo_setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/__init__.py b/easybuild/toolchains/__init__.py index 9c05df7c2b..b0a226866d 100644 --- a/easybuild/toolchains/__init__.py +++ b/easybuild/toolchains/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/cgmpich.py b/easybuild/toolchains/cgmpich.py index af3a4e9dbf..dd7f96b3aa 100644 --- a/easybuild/toolchains/cgmpich.py +++ b/easybuild/toolchains/cgmpich.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgmpolf.py b/easybuild/toolchains/cgmpolf.py index 86668d28bd..fa6a958ab7 100644 --- a/easybuild/toolchains/cgmpolf.py +++ b/easybuild/toolchains/cgmpolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgmvapich2.py b/easybuild/toolchains/cgmvapich2.py index 61a764c40e..3ef8902633 100644 --- a/easybuild/toolchains/cgmvapich2.py +++ b/easybuild/toolchains/cgmvapich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgmvolf.py b/easybuild/toolchains/cgmvolf.py index 682853b2be..dcfc7740f2 100644 --- a/easybuild/toolchains/cgmvolf.py +++ b/easybuild/toolchains/cgmvolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgompi.py b/easybuild/toolchains/cgompi.py index b39c6a0c23..c988a62db3 100644 --- a/easybuild/toolchains/cgompi.py +++ b/easybuild/toolchains/cgompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgoolf.py b/easybuild/toolchains/cgoolf.py index a02cbb7b94..abc073c41b 100644 --- a/easybuild/toolchains/cgoolf.py +++ b/easybuild/toolchains/cgoolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/clanggcc.py b/easybuild/toolchains/clanggcc.py index 17208be047..1fe2beb801 100644 --- a/easybuild/toolchains/clanggcc.py +++ b/easybuild/toolchains/clanggcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/compiler/__init__.py b/easybuild/toolchains/compiler/__init__.py index 79d806fe7e..95a05537f5 100644 --- a/easybuild/toolchains/compiler/__init__.py +++ b/easybuild/toolchains/compiler/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/clang.py b/easybuild/toolchains/compiler/clang.py index d5b676bde1..4192561fff 100644 --- a/easybuild/toolchains/compiler/clang.py +++ b/easybuild/toolchains/compiler/clang.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. @@ -32,6 +32,7 @@ """ import easybuild.tools.systemtools as systemtools +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.compiler import Compiler @@ -98,6 +99,5 @@ def _set_compiler_vars(self): super(Clang, self)._set_compiler_vars() if self.options.get('32bit', None): - self.log.raiseException("_set_compiler_vars: 32bit set, but no support yet for " \ - "32bit Clang in EasyBuild") + raise EasyBuildError("_set_compiler_vars: 32bit set, but no support yet for 32bit Clang in EasyBuild") diff --git a/easybuild/toolchains/compiler/craype.py b/easybuild/toolchains/compiler/craype.py new file mode 100644 index 0000000000..8250c2b9f5 --- /dev/null +++ b/easybuild/toolchains/compiler/craype.py @@ -0,0 +1,158 @@ +## +# Copyright 2014-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Support for the Cray Programming Environment (craype) compiler drivers (aka cc, CC, ftn). + +The basic concept is that the compiler driver knows how to invoke the true underlying +compiler with the compiler's specific options tuned to Cray systems. + +That means that certain defaults are set that are specific to Cray's computers. + +The compiler drivers are quite similar to EB toolchains as they include +linker and compiler directives to use the Cray libraries for their MPI (and network drivers) +Cray's LibSci (BLAS/LAPACK et al), FFT library, etc. + +@author: Petar Forai (IMP/IMBA, Austria) +@author: Kenneth Hoste (Ghent University) +""" +import easybuild.tools.environment as env +from easybuild.toolchains.compiler.gcc import TC_CONSTANT_GCC, Gcc +from easybuild.toolchains.compiler.inteliccifort import TC_CONSTANT_INTELCOMP, IntelIccIfort +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import build_option +from easybuild.tools.toolchain.compiler import Compiler + + +TC_CONSTANT_CRAYPE = "CrayPE" +TC_CONSTANT_CRAYCE = "CrayCE" + + +class CrayPECompiler(Compiler): + """Generic support for using Cray compiler drivers.""" + TOOLCHAIN_FAMILY = TC_CONSTANT_CRAYPE + + # compiler module name is PrgEnv, suffix name depends on CrayPE flavor (gnu, intel, cray) + COMPILER_MODULE_NAME = None + # compiler family depends on CrayPE flavor + COMPILER_FAMILY = None + + COMPILER_UNIQUE_OPTS = { + 'dynamic': (False, "Generate dynamically linked executable"), + 'mpich-mt': (False, "Directs the driver to link in an alternate version of the Cray-MPICH library which \ + provides fine-grained multi-threading support to applications that perform \ + MPI operations within threaded regions."), + 'optarch': (False, "Enable architecture optimizations"), + 'verbose': (True, "Verbose output"), + } + + COMPILER_UNIQUE_OPTION_MAP = { + # handle shared and dynamic always via $CRAYPE_LINK_TYPE environment variable, don't pass flags to wrapper + 'shared': '', + 'dynamic': '', + 'verbose': 'craype-verbose', + 'mpich-mt': 'craympich-mt', + } + + COMPILER_CC = 'cc' + COMPILER_CXX = 'CC' + + COMPILER_F77 = 'ftn' + COMPILER_F90 = 'ftn' + + # suffix for PrgEnv module that matches this toolchain + # e.g. 'gnu' => 'PrgEnv-gnu/' + PRGENV_MODULE_NAME_SUFFIX = None + + # template for craype module (determines code generator backend of Cray compiler wrappers) + CRAYPE_MODULE_NAME_TEMPLATE = 'craype-%(optarch)s' + + def __init__(self, *args, **kwargs): + """Constructor.""" + super(CrayPECompiler, self).__init__(*args, **kwargs) + # 'register' additional toolchain options that correspond to a compiler flag + self.COMPILER_FLAGS.extend(['dynamic', 'mpich-mt']) + + # use name of PrgEnv module as name of module that provides compiler + self.COMPILER_MODULE_NAME = ['PrgEnv-%s' % self.PRGENV_MODULE_NAME_SUFFIX] + + def _set_optimal_architecture(self): + """Load craype module specified via 'optarch' build option.""" + optarch = build_option('optarch') + if optarch is None: + raise EasyBuildError("Don't know which 'craype' module to load, 'optarch' build option is unspecified.") + else: + craype_mod_name = self.CRAYPE_MODULE_NAME_TEMPLATE % {'optarch': optarch} + if self.modules_tool.exist([craype_mod_name])[0]: + self.modules_tool.load([craype_mod_name]) + else: + raise EasyBuildError("Necessary craype module with name '%s' is not available (optarch: '%s')", + craype_mod_name, optarch) + + # no compiler flag when optarch toolchain option is enabled + self.options.options_map['optarch'] = '' + + def prepare(self, *args, **kwargs): + """Prepare to use this toolchain; define $CRAYPE_LINK_TYPE if 'dynamic' toolchain option is enabled.""" + super(CrayPECompiler, self).prepare(*args, **kwargs) + + if self.options['dynamic'] or self.options['shared']: + self.log.debug("Enabling building of shared libs/dynamically linked executables via $CRAYPE_LINK_TYPE") + env.setvar('CRAYPE_LINK_TYPE', 'dynamic') + + +class CrayPEGCC(CrayPECompiler): + """Support for using the Cray GNU compiler wrappers.""" + PRGENV_MODULE_NAME_SUFFIX = 'gnu' # PrgEnv-gnu + COMPILER_FAMILY = TC_CONSTANT_GCC + + def __init__(self, *args, **kwargs): + """CrayPEGCC constructor.""" + super(CrayPEGCC, self).__init__(*args, **kwargs) + for precflag in self.COMPILER_PREC_FLAGS: + self.COMPILER_UNIQUE_OPTION_MAP[precflag] = Gcc.COMPILER_UNIQUE_OPTION_MAP[precflag] + + +class CrayPEIntel(CrayPECompiler): + """Support for using the Cray Intel compiler wrappers.""" + PRGENV_MODULE_NAME_SUFFIX = 'intel' # PrgEnv-intel + COMPILER_FAMILY = TC_CONSTANT_INTELCOMP + + def __init__(self, *args, **kwargs): + """CrayPEIntel constructor.""" + super(CrayPEIntel, self).__init__(*args, **kwargs) + for precflag in self.COMPILER_PREC_FLAGS: + self.COMPILER_UNIQUE_OPTION_MAP[precflag] = IntelIccIfort.COMPILER_UNIQUE_OPTION_MAP[precflag] + + +class CrayPECray(CrayPECompiler): + """Support for using the Cray CCE compiler wrappers.""" + PRGENV_MODULE_NAME_SUFFIX = 'cray' # PrgEnv-cray + COMPILER_FAMILY = TC_CONSTANT_CRAYCE + + def __init__(self, *args, **kwargs): + """CrayPEIntel constructor.""" + super(CrayPECray, self).__init__(*args, **kwargs) + for precflag in self.COMPILER_PREC_FLAGS: + self.COMPILER_UNIQUE_OPTION_MAP[precflag] = [] diff --git a/easybuild/toolchains/compiler/cuda.py b/easybuild/toolchains/compiler/cuda.py index 7304b8b1f2..9397a9a584 100644 --- a/easybuild/toolchains/compiler/cuda.py +++ b/easybuild/toolchains/compiler/cuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/dummycompiler.py b/easybuild/toolchains/compiler/dummycompiler.py index f99b687c49..9296ac85d5 100644 --- a/easybuild/toolchains/compiler/dummycompiler.py +++ b/easybuild/toolchains/compiler/dummycompiler.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index 3ec1888438..bdd4fb7001 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -30,6 +30,7 @@ """ import easybuild.tools.systemtools as systemtools +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.compiler import Compiler @@ -83,8 +84,7 @@ def _set_compiler_vars(self): super(Gcc, self)._set_compiler_vars() if self.options.get('32bit', None): - self.log.raiseException("_set_compiler_vars: 32bit set, but no support yet for " \ - "32bit GCC in EasyBuild") + raise EasyBuildError("_set_compiler_vars: 32bit set, but no support yet for 32bit GCC in EasyBuild") # to get rid of lots of problems with libgfortranbegin # or remove the system gcc-gfortran diff --git a/easybuild/toolchains/compiler/inteliccifort.py b/easybuild/toolchains/compiler/inteliccifort.py index 3804343c6d..2e071033a2 100644 --- a/easybuild/toolchains/compiler/inteliccifort.py +++ b/easybuild/toolchains/compiler/inteliccifort.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -32,6 +32,7 @@ from distutils.version import LooseVersion import easybuild.tools.systemtools as systemtools +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.compiler import Compiler @@ -93,14 +94,15 @@ def _set_compiler_vars(self): super(IntelIccIfort, self)._set_compiler_vars() if not ('icc' in self.COMPILER_MODULE_NAME and 'ifort' in self.COMPILER_MODULE_NAME): - self.log.raiseException("_set_compiler_vars: missing icc and/or ifort from COMPILER_MODULE_NAME %s" % self.COMPILER_MODULE_NAME) + raise EasyBuildError("_set_compiler_vars: missing icc and/or ifort from COMPILER_MODULE_NAME %s", + self.COMPILER_MODULE_NAME) icc_root, _ = self.get_software_root(self.COMPILER_MODULE_NAME) icc_version, ifort_version = self.get_software_version(self.COMPILER_MODULE_NAME) if not ifort_version == icc_version: - msg = "_set_compiler_vars: mismatch between icc version %s and ifort version %s" - self.log.raiseException(msg % (icc_version, ifort_version)) + raise EasyBuildError("_set_compiler_vars: mismatch between icc version %s and ifort version %s", + icc_version, ifort_version) if LooseVersion(icc_version) < LooseVersion('2011'): self.LIB_MULTITHREAD.insert(1, "guide") diff --git a/easybuild/toolchains/craycce.py b/easybuild/toolchains/craycce.py new file mode 100644 index 0000000000..db86c0b266 --- /dev/null +++ b/easybuild/toolchains/craycce.py @@ -0,0 +1,44 @@ +## +# Copyright 2014-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +CrayCCE toolchain: Cray compilers (CCE) and MPI via Cray compiler drivers + LibSci (PrgEnv-cray) and Cray FFTW + +@author: Petar Forai (IMP/IMBA, Austria) +@author: Kenneth Hoste (Ghent University) +""" +from easybuild.toolchains.compiler.craype import CrayPECray +from easybuild.toolchains.fft.crayfftw import CrayFFTW +from easybuild.toolchains.linalg.libsci import LibSci +from easybuild.toolchains.mpi.craympich import CrayMPICH + + +class CrayCCE(CrayPECray, CrayMPICH, LibSci, CrayFFTW): + """Compiler toolchain for Cray Programming Environment for Cray Compiling Environment (CCE) (PrgEnv-cray).""" + NAME = 'CrayCCE' + + def prepare(self, *args, **kwargs): + """Prepare to use this toolchain; marked as experimental.""" + self.log.experimental("Using %s toolchain", self.NAME) + super(CrayCCE, self).prepare(*args, **kwargs) diff --git a/easybuild/toolchains/craygnu.py b/easybuild/toolchains/craygnu.py new file mode 100644 index 0000000000..9deef321ec --- /dev/null +++ b/easybuild/toolchains/craygnu.py @@ -0,0 +1,44 @@ +## +# Copyright 2014-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +CrayGNU toolchain: GCC and MPI via Cray compiler drivers + LibSci (PrgEnv-gnu) and Cray FFTW + +@author: Petar Forai (IMP/IMBA, Austria) +@author: Kenneth Hoste (Ghent University) +""" +from easybuild.toolchains.compiler.craype import CrayPEGCC +from easybuild.toolchains.fft.crayfftw import CrayFFTW +from easybuild.toolchains.linalg.libsci import LibSci +from easybuild.toolchains.mpi.craympich import CrayMPICH + + +class CrayGNU(CrayPEGCC, CrayMPICH, LibSci, CrayFFTW): + """Compiler toolchain for Cray Programming Environment for GCC compilers (PrgEnv-gnu).""" + NAME = 'CrayGNU' + + def prepare(self, *args, **kwargs): + """Prepare to use this toolchain; marked as experimental.""" + self.log.experimental("Using %s toolchain", self.NAME) + super(CrayGNU, self).prepare(*args, **kwargs) diff --git a/easybuild/toolchains/crayintel.py b/easybuild/toolchains/crayintel.py new file mode 100644 index 0000000000..92ee0e9b1a --- /dev/null +++ b/easybuild/toolchains/crayintel.py @@ -0,0 +1,44 @@ +## +# Copyright 2014-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +CrayIntel toolchain: Intel compilers and MPI via Cray compiler drivers + LibSci (PrgEnv-intel) and Cray FFTW + +@author: Petar Forai (IMP/IMBA, Austria) +@author: Kenneth Hoste (Ghent University) +""" +from easybuild.toolchains.compiler.craype import CrayPEIntel +from easybuild.toolchains.fft.crayfftw import CrayFFTW +from easybuild.toolchains.linalg.libsci import LibSci +from easybuild.toolchains.mpi.craympich import CrayMPICH + + +class CrayIntel(CrayPEIntel, CrayMPICH, LibSci, CrayFFTW): + """Compiler toolchain for Cray Programming Environment for Intel compilers (PrgEnv-intel).""" + NAME = 'CrayIntel' + + def prepare(self, *args, **kwargs): + """Prepare to use this toolchain; marked as experimental.""" + self.log.experimental("Using %s toolchain", self.NAME) + super(CrayIntel, self).prepare(*args, **kwargs) diff --git a/easybuild/toolchains/dummy.py b/easybuild/toolchains/dummy.py index c7791744fe..c641b89952 100644 --- a/easybuild/toolchains/dummy.py +++ b/easybuild/toolchains/dummy.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fft/__init__.py b/easybuild/toolchains/fft/__init__.py index 8490061d53..0e74f343d7 100644 --- a/easybuild/toolchains/fft/__init__.py +++ b/easybuild/toolchains/fft/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fft/crayfftw.py b/easybuild/toolchains/fft/crayfftw.py new file mode 100644 index 0000000000..b575cbc9d9 --- /dev/null +++ b/easybuild/toolchains/fft/crayfftw.py @@ -0,0 +1,71 @@ +## +# Copyright 2014-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Support for Cray FFTW. + +@author: Petar Forai (IMP/IMBA, Austria) +@author: Kenneth Hoste (Ghent University) +""" +import os + +from easybuild.toolchains.fft.fftw import Fftw +from easybuild.tools.build_log import EasyBuildError + + +class CrayFFTW(Fftw): + """Support for Cray FFTW.""" + # FFT support, via Cray-provided fftw module + FFT_MODULE_NAME = ['fftw'] + + def _get_software_root(self, name): + """Get install prefix for specified software name; special treatment for Cray modules.""" + if name == 'fftw': + # Cray-provided fftw module + env_var = 'FFTW_INC' + incdir = os.getenv(env_var, None) + if incdir is None: + raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_var) + else: + root = os.path.dirname(incdir) + self.log.debug("Obtained install prefix for %s via $%s: %s", name, env_var, root) + else: + root = super(CrayFFTW, self)._get_software_root(name) + + return root + + def _get_software_version(self, name): + """Get version for specified software name; special treatment for Cray modules.""" + if name == 'fftw': + # Cray-provided fftw module + env_var = 'FFTW_VERSION' + ver = os.getenv(env_var, None) + if ver is None: + raise EasyBuildError("Failed to determine version for %s via $%s", name, env_var) + else: + self.log.debug("Obtained version for %s via $%s: %s", name, env_var, ver) + else: + ver = super(CrayFFTW, self)._get_software_version(name) + + return ver diff --git a/easybuild/toolchains/fft/fftw.py b/easybuild/toolchains/fft/fftw.py index 1d36693733..1e9a9c3825 100644 --- a/easybuild/toolchains/fft/fftw.py +++ b/easybuild/toolchains/fft/fftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -31,6 +31,7 @@ from distutils.version import LooseVersion +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.fft import Fft @@ -44,7 +45,7 @@ def _set_fftw_variables(self): suffix = '' version = self.get_software_version(self.FFT_MODULE_NAME)[0] if LooseVersion(version) < LooseVersion('2') or LooseVersion(version) >= LooseVersion('4'): - self.log.raiseException("_set_fft_variables: FFTW unsupported version %s (major should be 2 or 3)" % version) + raise EasyBuildError("_set_fft_variables: FFTW unsupported version %s (major should be 2 or 3)", version) elif LooseVersion(version) > LooseVersion('2'): suffix = '3' diff --git a/easybuild/toolchains/fft/intelfftw.py b/easybuild/toolchains/fft/intelfftw.py index a6eb1e1eaa..94b53c7372 100644 --- a/easybuild/toolchains/fft/intelfftw.py +++ b/easybuild/toolchains/fft/intelfftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -31,6 +31,7 @@ import os from distutils.version import LooseVersion +from easybuild.tools.build_log import EasyBuildError from easybuild.toolchains.fft.fftw import Fftw from easybuild.tools.modules import get_software_root, get_software_version @@ -45,7 +46,7 @@ class IntelFFTW(Fftw): def _set_fftw_variables(self): if not hasattr(self, 'BLAS_LIB_DIR'): - self.log.raiseException("_set_fftw_variables: IntelFFT based on IntelMKL (no BLAS_LIB_DIR found)") + raise EasyBuildError("_set_fftw_variables: IntelFFT based on IntelMKL (no BLAS_LIB_DIR found)") imklver = get_software_version(self.FFT_MODULE_NAME[0]) @@ -60,7 +61,7 @@ def _set_fftw_variables(self): if get_software_root('GCC'): compsuff = '_gnu' else: - self.log.error("Not using Intel compilers or GCC, don't know compiler suffix for FFTW libraries.") + raise EasyBuildError("Not using Intel compilers or GCC, don't know compiler suffix for FFTW libraries.") fftw_libs = ["fftw3xc%s%s" % (compsuff, picsuff)] if self.options['usempi']: @@ -89,6 +90,5 @@ def _set_fftw_variables(self): if all([fftw_lib_exists(lib) for lib in check_fftw_libs]): self.FFT_LIB = fftw_libs else: - tup = (check_fftw_libs, fft_lib_dirs) - msg = "Not all FFTW interface libraries %s are found in %s, can't set FFT_LIB." % tup - self.log.error(msg) + raise EasyBuildError("Not all FFTW interface libraries %s are found in %s, can't set FFT_LIB.", + check_fftw_libs, fft_lib_dirs) diff --git a/easybuild/toolchains/foss.py b/easybuild/toolchains/foss.py index d67dd5267c..1a8b7c69c4 100755 --- a/easybuild/toolchains/foss.py +++ b/easybuild/toolchains/foss.py @@ -1,5 +1,5 @@ ## -# Copyright 2013 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gcc.py b/easybuild/toolchains/gcc.py index 749dd7cd9c..60115094d7 100644 --- a/easybuild/toolchains/gcc.py +++ b/easybuild/toolchains/gcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gcccuda.py b/easybuild/toolchains/gcccuda.py index 9bd72024bb..c76c53d200 100644 --- a/easybuild/toolchains/gcccuda.py +++ b/easybuild/toolchains/gcccuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gimkl.py b/easybuild/toolchains/gimkl.py index 894eda83af..a9ae8db870 100644 --- a/easybuild/toolchains/gimkl.py +++ b/easybuild/toolchains/gimkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gimpi.py b/easybuild/toolchains/gimpi.py index 0ac2345266..fb525d962b 100644 --- a/easybuild/toolchains/gimpi.py +++ b/easybuild/toolchains/gimpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmacml.py b/easybuild/toolchains/gmacml.py index b76b3e3838..d64d715c90 100644 --- a/easybuild/toolchains/gmacml.py +++ b/easybuild/toolchains/gmacml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmpich.py b/easybuild/toolchains/gmpich.py index 4527db29f2..8640dc68ca 100644 --- a/easybuild/toolchains/gmpich.py +++ b/easybuild/toolchains/gmpich.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmpich2.py b/easybuild/toolchains/gmpich2.py index caf6e0af5b..b70848a354 100644 --- a/easybuild/toolchains/gmpich2.py +++ b/easybuild/toolchains/gmpich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmpolf.py b/easybuild/toolchains/gmpolf.py index 35a0d0b899..e849650be2 100644 --- a/easybuild/toolchains/gmpolf.py +++ b/easybuild/toolchains/gmpolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/gmvapich2.py b/easybuild/toolchains/gmvapich2.py index 03c4b94366..438d209d07 100644 --- a/easybuild/toolchains/gmvapich2.py +++ b/easybuild/toolchains/gmvapich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmvolf.py b/easybuild/toolchains/gmvolf.py index 9e8d9ee34d..90ba0e39e1 100644 --- a/easybuild/toolchains/gmvolf.py +++ b/easybuild/toolchains/gmvolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/gnu.py b/easybuild/toolchains/gnu.py new file mode 100644 index 0000000000..8b7f97ad70 --- /dev/null +++ b/easybuild/toolchains/gnu.py @@ -0,0 +1,36 @@ +## +# Copyright 2012-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for GCC compiler toolchain. + +@author: Kenneth Hoste (Ghent University) +""" + +from easybuild.toolchains.compiler.gcc import Gcc + + +class GNU(Gcc): + """Compiler-only toolchain, including only GCC and binutils.""" + NAME = 'GNU' diff --git a/easybuild/toolchains/goalf.py b/easybuild/toolchains/goalf.py index 8c71ee927b..28a087887b 100644 --- a/easybuild/toolchains/goalf.py +++ b/easybuild/toolchains/goalf.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gompi.py b/easybuild/toolchains/gompi.py index 18698f9cd7..cbea82b54c 100644 --- a/easybuild/toolchains/gompi.py +++ b/easybuild/toolchains/gompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gompic.py b/easybuild/toolchains/gompic.py index 0a331d5daf..984ea26272 100644 --- a/easybuild/toolchains/gompic.py +++ b/easybuild/toolchains/gompic.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/goolf.py b/easybuild/toolchains/goolf.py index 2970099d71..6b51b34e66 100644 --- a/easybuild/toolchains/goolf.py +++ b/easybuild/toolchains/goolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/goolfc.py b/easybuild/toolchains/goolfc.py index 96e2f49574..184272a6f0 100644 --- a/easybuild/toolchains/goolfc.py +++ b/easybuild/toolchains/goolfc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gpsmpi.py b/easybuild/toolchains/gpsmpi.py index b1dc8e0ab5..0109d0b389 100644 --- a/easybuild/toolchains/gpsmpi.py +++ b/easybuild/toolchains/gpsmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gpsolf.py b/easybuild/toolchains/gpsolf.py index f454620df8..592556f148 100644 --- a/easybuild/toolchains/gpsolf.py +++ b/easybuild/toolchains/gpsolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/gqacml.py b/easybuild/toolchains/gqacml.py index 9731a5bfc7..f7426a9055 100644 --- a/easybuild/toolchains/gqacml.py +++ b/easybuild/toolchains/gqacml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iccifort.py b/easybuild/toolchains/iccifort.py index 748fc8b4b1..4c688d4eb5 100644 --- a/easybuild/toolchains/iccifort.py +++ b/easybuild/toolchains/iccifort.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ictce.py b/easybuild/toolchains/ictce.py index 9cf59a1eaf..468741937d 100644 --- a/easybuild/toolchains/ictce.py +++ b/easybuild/toolchains/ictce.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iimpi.py b/easybuild/toolchains/iimpi.py index 781223ca34..ef5dddeafe 100644 --- a/easybuild/toolchains/iimpi.py +++ b/easybuild/toolchains/iimpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iiqmpi.py b/easybuild/toolchains/iiqmpi.py index 58141a9471..4ef089d181 100644 --- a/easybuild/toolchains/iiqmpi.py +++ b/easybuild/toolchains/iiqmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/impich.py b/easybuild/toolchains/impich.py index efe9c513a2..f837f77325 100644 --- a/easybuild/toolchains/impich.py +++ b/easybuild/toolchains/impich.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/impmkl.py b/easybuild/toolchains/impmkl.py index 383753d1db..7c8f3eaad2 100644 --- a/easybuild/toolchains/impmkl.py +++ b/easybuild/toolchains/impmkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/intel-para.py b/easybuild/toolchains/intel-para.py index 35e23da99a..31e12d638b 100644 --- a/easybuild/toolchains/intel-para.py +++ b/easybuild/toolchains/intel-para.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2013 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/intel.py b/easybuild/toolchains/intel.py index 25a88b6100..02c2bd6618 100644 --- a/easybuild/toolchains/intel.py +++ b/easybuild/toolchains/intel.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2013 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iomkl.py b/easybuild/toolchains/iomkl.py index 57f41d7e35..a13fb71cea 100644 --- a/easybuild/toolchains/iomkl.py +++ b/easybuild/toolchains/iomkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iompi.py b/easybuild/toolchains/iompi.py index fd757a3c92..b64cc8e759 100644 --- a/easybuild/toolchains/iompi.py +++ b/easybuild/toolchains/iompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ipsmpi.py b/easybuild/toolchains/ipsmpi.py index 8b99018284..de8b8806a1 100644 --- a/easybuild/toolchains/ipsmpi.py +++ b/easybuild/toolchains/ipsmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iqacml.py b/easybuild/toolchains/iqacml.py index dc6505112a..9529746985 100644 --- a/easybuild/toolchains/iqacml.py +++ b/easybuild/toolchains/iqacml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ismkl.py b/easybuild/toolchains/ismkl.py index de3c965a46..41afc4a4f9 100644 --- a/easybuild/toolchains/ismkl.py +++ b/easybuild/toolchains/ismkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/__init__.py b/easybuild/toolchains/linalg/__init__.py index 9fdd109ac1..26041ec702 100644 --- a/easybuild/toolchains/linalg/__init__.py +++ b/easybuild/toolchains/linalg/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/acml.py b/easybuild/toolchains/linalg/acml.py index 62c18c0ef7..9594fa428a 100644 --- a/easybuild/toolchains/linalg/acml.py +++ b/easybuild/toolchains/linalg/acml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,6 +34,7 @@ from easybuild.toolchains.compiler.inteliccifort import TC_CONSTANT_INTELCOMP from easybuild.toolchains.compiler.gcc import TC_CONSTANT_GCC +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.linalg import LinAlg @@ -60,7 +61,7 @@ class Acml(LinAlg): def _set_blas_variables(self): """Fix the map a bit""" if self.options.get('32bit', None): - self.log.raiseException("_set_blas_variables: 32bit ACML not (yet) supported") + raise EasyBuildError("_set_blas_variables: 32bit ACML not (yet) supported") try: for root in self.get_software_root(self.BLAS_MODULE_NAME): subdirs = self.ACML_SUBDIRS_MAP[self.COMPILER_FAMILY] @@ -69,8 +70,8 @@ def _set_blas_variables(self): incdirs = [os.path.join(x, 'include') for x in subdirs] self.variables.append_exists('CPPFLAGS', root, incdirs, append_all=True) except: - self.log.raiseException(("_set_blas_variables: ACML set LDFLAGS/CPPFLAGS unknown entry in ACML_SUBDIRS_MAP" - " with compiler family %s") % self.COMPILER_FAMILY) + raise EasyBuildError("_set_blas_variables: ACML set LDFLAGS/CPPFLAGS unknown entry in ACML_SUBDIRS_MAP " + "with compiler family %s", self.COMPILER_FAMILY) # version before 5.x still featured the acml_mv library ver = self.get_software_version(self.BLAS_MODULE_NAME)[0] diff --git a/easybuild/toolchains/linalg/atlas.py b/easybuild/toolchains/linalg/atlas.py index 33313b4ae6..f1320600ac 100644 --- a/easybuild/toolchains/linalg/atlas.py +++ b/easybuild/toolchains/linalg/atlas.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/blacs.py b/easybuild/toolchains/linalg/blacs.py index 9bdebdf2e8..e7e708e392 100644 --- a/easybuild/toolchains/linalg/blacs.py +++ b/easybuild/toolchains/linalg/blacs.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/flame.py b/easybuild/toolchains/linalg/flame.py index e48848649e..e276e67327 100644 --- a/easybuild/toolchains/linalg/flame.py +++ b/easybuild/toolchains/linalg/flame.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/gotoblas.py b/easybuild/toolchains/linalg/gotoblas.py index 48719dcf73..c196d2fe5b 100644 --- a/easybuild/toolchains/linalg/gotoblas.py +++ b/easybuild/toolchains/linalg/gotoblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py index af8cca4739..6d1886212a 100644 --- a/easybuild/toolchains/linalg/intelmkl.py +++ b/easybuild/toolchains/linalg/intelmkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -38,6 +38,7 @@ from easybuild.toolchains.mpi.mpich2 import TC_CONSTANT_MPICH2 from easybuild.toolchains.mpi.mvapich2 import TC_CONSTANT_MVAPICH2 from easybuild.toolchains.mpi.openmpi import TC_CONSTANT_OPENMPI +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.linalg import LinAlg @@ -84,8 +85,8 @@ def _set_blas_variables(self): "interface": interfacemap[self.COMPILER_FAMILY], }) except: - self.log.raiseException(("_set_blas_variables: interface unsupported combination" - " with MPI family %s") % self.COMPILER_FAMILY) + raise EasyBuildError("_set_blas_variables: interface unsupported combination with MPI family %s", + self.COMPILER_FAMILY) interfacemap_mt = { TC_CONSTANT_INTELCOMP: 'intel', @@ -94,8 +95,8 @@ def _set_blas_variables(self): try: self.BLAS_LIB_MAP.update({"interface_mt":interfacemap_mt[self.COMPILER_FAMILY]}) except: - self.log.raiseException(("_set_blas_variables: interface_mt unsupported combination " - "with compiler family %s") % self.COMPILER_FAMILY) + raise EasyBuildError("_set_blas_variables: interface_mt unsupported combination with compiler family %s", + self.COMPILER_FAMILY) if self.options.get('32bit', None): @@ -117,8 +118,8 @@ def _set_blas_variables(self): self.BLAS_INCLUDE_DIR = ['include'] else: if self.options.get('32bit', None): - self.log.raiseException(("_set_blas_variables: 32-bit libraries not supported yet " - "for IMKL v%s (> v10.3)") % found_version) + raise EasyBuildError("_set_blas_variables: 32-bit libraries not supported yet for IMKL v%s (> v10.3)", + found_version) else: self.BLAS_LIB_DIR = ['mkl/lib/intel64', 'compiler/lib/intel64' ] @@ -140,8 +141,8 @@ def _set_blacs_variables(self): try: self.BLACS_LIB_MAP.update({'mpi': mpimap[self.MPI_FAMILY]}) except: - self.log.raiseException(("_set_blacs_variables: mpi unsupported combination with" - " MPI family %s") % self.MPI_FAMILY) + raise EasyBuildError("_set_blacs_variables: mpi unsupported combination with MPI family %s", + self.MPI_FAMILY) self.BLACS_LIB_DIR = self.BLAS_LIB_DIR self.BLACS_INCLUDE_DIR = self.BLAS_INCLUDE_DIR @@ -166,4 +167,3 @@ def _set_scalapack_variables(self): self.SCALAPACK_INCLUDE_DIR = self.BLAS_INCLUDE_DIR super(IntelMKL, self)._set_scalapack_variables() - diff --git a/easybuild/toolchains/linalg/lapack.py b/easybuild/toolchains/linalg/lapack.py index d8933bb158..0a95b9f456 100644 --- a/easybuild/toolchains/linalg/lapack.py +++ b/easybuild/toolchains/linalg/lapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/libsci.py b/easybuild/toolchains/linalg/libsci.py new file mode 100644 index 0000000000..d332b8d3d0 --- /dev/null +++ b/easybuild/toolchains/linalg/libsci.py @@ -0,0 +1,90 @@ +## +# Copyright 2014-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Support for Cray's LibSci library, which provides BLAS/LAPACK support. +cfr. https://www.nersc.gov/users/software/programming-libraries/math-libraries/libsci/ + +@author: Petar Forai (IMP/IMBA, Austria) +@author: Kenneth Hoste (Ghent University) +""" +import os + +from easybuild.tools.toolchain.linalg import LinAlg + + +CRAY_LIBSCI_MODULE_NAME = 'cray-libsci' + + +class LibSci(LinAlg): + """Support for Cray's LibSci library, which provides BLAS/LAPACK support.""" + # BLAS/LAPACK support + # via cray-libsci module, which gets loaded via the PrgEnv module + # see https://www.nersc.gov/users/software/programming-libraries/math-libraries/libsci/ + BLAS_MODULE_NAME = [CRAY_LIBSCI_MODULE_NAME] + + # no need to specify libraries, compiler driver takes care of linking the right libraries + # FIXME: need to revisit this, on numpy we ended up with a serial BLAS through the wrapper. + BLAS_LIB = [] + BLAS_LIB_MT = [] + + LAPACK_MODULE_NAME = [CRAY_LIBSCI_MODULE_NAME] + LAPACK_IS_BLAS = True + + BLACS_MODULE_NAME = [] + SCALAPACK_MODULE_NAME = [] + + def _get_software_root(self, name): + """Get install prefix for specified software name; special treatment for Cray modules.""" + if name == 'cray-libsci': + # Cray-provided LibSci module + env_var = 'CRAY_LIBSCI_PREFIX_DIR' + root = os.getenv(env_var, None) + if root is None: + raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_var) + else: + self.log.debug("Obtained install prefix for %s via $%s: %s", name, env_var, root) + else: + root = super(LibSci, self)._get_software_root(name) + + return root + + def _set_blacs_variables(self): + """Skip setting BLACS related variables""" + pass + + def _set_scalapack_variables(self): + """Skip setting ScaLAPACK related variables""" + pass + + def definition(self): + """ + Filter BLAS module from toolchain definition. + The cray-libsci module is loaded indirectly (and versionless) via the PrgEnv module, + and thus is not a direct toolchain component. + """ + tc_def = super(LibSci, self).definition() + tc_def['BLAS'] = [] + tc_def['LAPACK'] = [] + return tc_def diff --git a/easybuild/toolchains/linalg/openblas.py b/easybuild/toolchains/linalg/openblas.py index 798c63e7ac..1d1c6d4790 100644 --- a/easybuild/toolchains/linalg/openblas.py +++ b/easybuild/toolchains/linalg/openblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/scalapack.py b/easybuild/toolchains/linalg/scalapack.py index b3859346c2..eea44b1b45 100644 --- a/easybuild/toolchains/linalg/scalapack.py +++ b/easybuild/toolchains/linalg/scalapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/__init__.py b/easybuild/toolchains/mpi/__init__.py index 3f868dff91..e67a5cb97e 100644 --- a/easybuild/toolchains/mpi/__init__.py +++ b/easybuild/toolchains/mpi/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/craympich.py b/easybuild/toolchains/mpi/craympich.py new file mode 100644 index 0000000000..e2ec238008 --- /dev/null +++ b/easybuild/toolchains/mpi/craympich.py @@ -0,0 +1,79 @@ +## +# Copyright 2014-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +MPI support for the Cray Programming Environment (craype). + +@author: Petar Forai (IMP/IMBA, Austria) +@author: Kenneth Hoste (Ghent University) +""" +from easybuild.toolchains.compiler.craype import CrayPECompiler +from easybuild.toolchains.mpi.mpich import TC_CONSTANT_MPICH, TC_CONSTANT_MPI_TYPE_MPICH +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.toolchain.constants import COMPILER_VARIABLES, MPI_COMPILER_TEMPLATE, SEQ_COMPILER_TEMPLATE +from easybuild.tools.toolchain.mpi import Mpi + + +class CrayMPICH(Mpi): + """Generic support for using Cray compiler wrappers""" + # MPI support + # no separate module, Cray compiler drivers always provide MPI support + MPI_MODULE_NAME = [] + MPI_FAMILY = TC_CONSTANT_MPICH + MPI_TYPE = TC_CONSTANT_MPI_TYPE_MPICH + + MPI_COMPILER_MPICC = CrayPECompiler.COMPILER_CC + MPI_COMPILER_MPICXX = CrayPECompiler.COMPILER_CXX + MPI_COMPILER_MPIF77 = CrayPECompiler.COMPILER_F77 + MPI_COMPILER_MPIF90 = CrayPECompiler.COMPILER_F90 + + # no MPI wrappers, so no need to specify serial compiler + MPI_SHARED_OPTION_MAP = { + '_opt_MPICC': '', + '_opt_MPICXX': '', + '_opt_MPIF77': '', + '_opt_MPIF90': '', + } + + def _set_mpi_compiler_variables(self): + """Set the MPI compiler variables""" + for var_tuple in COMPILER_VARIABLES: + c_var = var_tuple[0] # [1] is the description + var = MPI_COMPILER_TEMPLATE % {'c_var':c_var} + + value = getattr(self, 'MPI_COMPILER_%s' % var.upper(), None) + if value is None: + raise EasyBuildError("_set_mpi_compiler_variables: mpi compiler variable %s undefined", var) + self.variables.nappend_el(var, value) + + if self.options.get('usempi', None): + var_seq = SEQ_COMPILER_TEMPLATE % {'c_var': c_var} + seq_comp = self.variables[c_var] + self.log.debug('_set_mpi_compiler_variables: usempi set: defining %s as %s', var_seq, seq_comp) + self.variables[var_seq] = seq_comp + + if self.options.get('cciscxx', None): + self.log.debug("_set_mpi_compiler_variables: cciscxx set: switching MPICXX %s for MPICC value %s" % + (self.variables['MPICXX'], self.variables['MPICC'])) + self.variables['MPICXX'] = self.variables['MPICC'] diff --git a/easybuild/toolchains/mpi/intelmpi.py b/easybuild/toolchains/mpi/intelmpi.py index f2bb79a926..a8fb512964 100644 --- a/easybuild/toolchains/mpi/intelmpi.py +++ b/easybuild/toolchains/mpi/intelmpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/mpich.py b/easybuild/toolchains/mpi/mpich.py index 8c61e8fdf7..b173148137 100644 --- a/easybuild/toolchains/mpi/mpich.py +++ b/easybuild/toolchains/mpi/mpich.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/mpich2.py b/easybuild/toolchains/mpi/mpich2.py index 21dc78de60..d68b7917d9 100644 --- a/easybuild/toolchains/mpi/mpich2.py +++ b/easybuild/toolchains/mpi/mpich2.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/mvapich2.py b/easybuild/toolchains/mpi/mvapich2.py index b9cbf3561a..000be188a1 100644 --- a/easybuild/toolchains/mpi/mvapich2.py +++ b/easybuild/toolchains/mpi/mvapich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/openmpi.py b/easybuild/toolchains/mpi/openmpi.py index 12e1a9ade0..8a2137e0f1 100644 --- a/easybuild/toolchains/mpi/openmpi.py +++ b/easybuild/toolchains/mpi/openmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/qlogicmpi.py b/easybuild/toolchains/mpi/qlogicmpi.py index 9118550664..2a7db39666 100644 --- a/easybuild/toolchains/mpi/qlogicmpi.py +++ b/easybuild/toolchains/mpi/qlogicmpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/__init__.py b/easybuild/tools/__init__.py index 15512e2f9c..6badf9fb34 100644 --- a/easybuild/tools/__init__.py +++ b/easybuild/tools/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/asyncprocess.py b/easybuild/tools/asyncprocess.py index 797478f70a..58e83de980 100644 --- a/easybuild/tools/asyncprocess.py +++ b/easybuild/tools/asyncprocess.py @@ -1,6 +1,6 @@ ## # Copyright 2005 Josiah Carlson -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # The Asynchronous Python Subprocess recipe was originally created by Josiah Carlson. # and released under the GPL v2 on March 14, 2012 diff --git a/easybuild/tools/build_details.py b/easybuild/tools/build_details.py index 44475ec9e8..4b6df10d39 100644 --- a/easybuild/tools/build_details.py +++ b/easybuild/tools/build_details.py @@ -1,4 +1,4 @@ -# Copyright 2014-2014 Ghent University +# Copyright 2014-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index 24178b9f84..3bf2a09527 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -36,6 +36,7 @@ import tempfile from copy import copy from vsc.utils import fancylogger +from vsc.utils.exceptions import LoggedException from easybuild.tools.version import VERSION @@ -52,18 +53,34 @@ DEPRECATED_DOC_URL = 'http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html' -class EasyBuildError(Exception): +class EasyBuildError(LoggedException): """ EasyBuildError is thrown when EasyBuild runs into something horribly wrong. """ - def __init__(self, msg): - Exception.__init__(self, msg) + LOC_INFO_TOP_PKG_NAMES = ['easybuild', 'vsc'] + LOC_INFO_LEVEL = 1 + + # use custom error logging method, to make sure EasyBuildError isn't being raised again to avoid infinite recursion + # only required because 'error' log method raises (should no longer be needed in EB v3.x) + LOGGING_METHOD_NAME = '_error_no_raise' + + def __init__(self, msg, *args): + """Constructor: initialise EasyBuildError instance.""" + if args: + msg = msg % args + LoggedException.__init__(self, msg) self.msg = msg def __str__(self): + """Return string representation of this EasyBuildError instance.""" return repr(self.msg) +def raise_easybuilderror(msg, *args): + """Raise EasyBuildError with given message, formatted by provided string arguments.""" + raise EasyBuildError(msg, *args) + + class EasyBuildLog(fancylogger.FancyLogger): """ The EasyBuild logger, with its own error and exception functions. @@ -94,7 +111,7 @@ def experimental(self, msg, *args, **kwargs): self.warning(msg, *args, **kwargs) else: msg = 'Experimental functionality. Behaviour might change/be removed later (use --experimental option to enable). ' + msg - self.error(msg, *args) + raise EasyBuildError(msg, *args) def deprecated(self, msg, max_ver): """Print deprecation warning or raise an EasyBuildError, depending on max version allowed.""" @@ -103,26 +120,45 @@ def deprecated(self, msg, max_ver): def nosupport(self, msg, ver): """Print error message for no longer supported behaviour, and raise an EasyBuildError.""" - self.error("NO LONGER SUPPORTED since v%s: %s; see %s for more information" % (ver, msg, DEPRECATED_DOC_URL)) + nosupport_msg = "NO LONGER SUPPORTED since v%s: %s; see %s for more information" + raise EasyBuildError(nosupport_msg, ver, msg, DEPRECATED_DOC_URL) def error(self, msg, *args, **kwargs): """Print error message and raise an EasyBuildError.""" - newMsg = ("EasyBuild crashed with an error %s: %s" - % (self.caller_info(), (msg % args))) - fancylogger.FancyLogger.error(self, newMsg, **kwargs) + ebmsg = "EasyBuild crashed with an error %s: " + msg + args = (self.caller_info(),) + args + + fancylogger.FancyLogger.error(self, ebmsg, *args, **kwargs) + if self.raiseError: - raise EasyBuildError(newMsg) + self.deprecated("Use 'raise EasyBuildError' rather than error() logging method that raises", '3.0') + raise EasyBuildError(ebmsg, *args) + + # FIXME: remove this when error() no longer raises EasyBuildError + def _error_no_raise(self, msg): + """Utility function to log an error with raising an exception.""" + + # make sure raising of error is disabled + orig_raise_error = self.raiseError + self.raiseError = False + + fancylogger.FancyLogger.error(self, msg) + + # reinstate previous raiseError setting + self.raiseError = orig_raise_error def exception(self, msg, *args): """Print exception message and raise EasyBuildError.""" # don't raise the exception from within error - newMsg = "EasyBuild encountered an exception %s: %s" % (self.caller_info(), msg) + ebmsg = "EasyBuild encountered an exception %s: " + msg + args = (self.caller_info(),) + args self.raiseError = False - fancylogger.FancyLogger.exception(self, newMsg, *args) + fancylogger.FancyLogger.exception(self, ebmsg, *args) self.raiseError = True - raise EasyBuildError(newMsg) + self.deprecated("Use 'raise EasyBuildError' rather than exception() logging method that raises", '3.0') + raise EasyBuildError(ebmsg, *args) # set format for logger @@ -199,7 +235,7 @@ def print_error(message, log=None, exitCode=1, opt_parser=None, exit_on_error=Tr sys.stderr.write("ERROR: %s\n" % message) sys.exit(exitCode) elif log is not None: - log.error(message) + raise EasyBuildError(message) def print_warning(message, silent=False): diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 0f3782dc11..e0352afd5f 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -40,12 +40,12 @@ import tempfile import time from vsc.utils import fancylogger -from vsc.utils.missing import nub, FrozenDictKnownKeys +from vsc.utils.missing import FrozenDictKnownKeys from vsc.utils.patterns import Singleton -import easybuild.tools.build_log # this import is required to obtain a correct (EasyBuild) logger! import easybuild.tools.environment as env -from easybuild.tools.environment import read_environment as _read_environment +from easybuild.tools import run +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.run import run_cmd @@ -54,6 +54,7 @@ DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log") DEFAULT_MNS = 'EasyBuildMNS' +DEFAULT_MODULE_SYNTAX = 'Tcl' DEFAULT_MODULES_TOOL = 'EnvironmentModulesC' DEFAULT_PATH_SUBDIRS = { 'buildpath': 'build', @@ -65,6 +66,7 @@ } DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild") DEFAULT_REPOSITORY = 'FileRepository' +DEFAULT_STRICT = run.WARN PREFERRED_JOB_BACKENDS = ('Pbs', 'GC3Pie') @@ -85,7 +87,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'download_timeout', 'dump_test_report', 'easyblock', + 'external_modules_metadata', 'filter_deps', + 'hide_deps', 'from_pr', 'github_user', 'group', @@ -107,6 +111,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'experimental', 'force', 'hidden', + 'module_only', 'robot', 'sequential', 'set_gid_bit', @@ -118,6 +123,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): True: [ 'cleanup_builddir', ], + DEFAULT_STRICT: [ + 'strict', + ], } # build option that do not have a perfectly matching command line option BUILD_OPTIONS_OTHER = { @@ -179,21 +187,24 @@ class ConfigurationVariables(FrozenDictKnownKeys): # list of known/required keys REQUIRED = [ - 'config', - 'prefix', 'buildpath', + 'config', 'installpath', + 'installpath_modules', + 'installpath_software', 'job_backend', - 'sourcepath', - 'repository', - 'repositorypath', 'logfile_format', - 'tmp_logdir', 'moduleclasses', + 'module_naming_scheme', + 'module_syntax', + 'modules_tool', + 'prefix', + 'repository', + 'repositorypath', + 'sourcepath', 'subdir_modules', 'subdir_software', - 'modules_tool', - 'module_naming_scheme', + 'tmp_logdir', ] KNOWN_KEYS = REQUIRED # KNOWN_KEYS must be defined for FrozenDictKnownKeys functionality @@ -204,8 +215,7 @@ def get_items_check_required(self): """ missing = [x for x in self.KNOWN_KEYS if not x in self] if len(missing) > 0: - msg = 'Cannot determine value for configuration variables %s. Please specify it.' % missing - self.log.error(msg) + raise EasyBuildError("Cannot determine value for configuration variables %s. Please specify it.", missing) return self.items() @@ -237,7 +247,7 @@ def init(options, config_options_dict): tmpdict['sourcepath'] = sourcepath.split(':') _log.debug("Converted source path ('%s') to a list of paths: %s" % (sourcepath, tmpdict['sourcepath'])) elif not isinstance(sourcepath, (tuple, list)): - _log.error("Value for sourcepath has invalid type (%s): %s" % (type(sourcepath), sourcepath)) + raise EasyBuildError("Value for sourcepath has invalid type (%s): %s", type(sourcepath), sourcepath) # initialize configuration variables (any future calls to ConfigurationVariables() will yield the same instance variables = ConfigurationVariables(tmpdict, ignore_unknown_keys=True) @@ -324,9 +334,22 @@ def install_path(typ=None): elif typ == 'mod': typ = 'modules' + known_types = ['modules', 'software'] + if typ not in known_types: + raise EasyBuildError("Unknown type specified in install_path(): %s (known: %s)", typ, ', '.join(known_types)) + variables = ConfigurationVariables() - suffix = variables['subdir_%s' % typ] - return os.path.join(variables['installpath'], suffix) + + key = 'installpath_%s' % typ + res = variables[key] + if res is None: + key = 'subdir_%s' % typ + res = os.path.join(variables['installpath'], variables[key]) + _log.debug("%s install path as specified by 'installpath' and '%s': %s", typ, key, res) + else: + _log.debug("%s install path as specified by '%s': %s", typ, key, res) + + return res def get_repository(): @@ -353,7 +376,7 @@ def get_modules_tool(): def get_module_naming_scheme(): """ - Return module naming scheme (EasyBuild, ...) + Return module naming scheme (EasyBuildMNS, HierarchicalMNS, ...) """ return ConfigurationVariables()['module_naming_scheme'] @@ -366,6 +389,13 @@ def get_job_backend(): return ConfigurationVariables().get('job_backend', None) +def get_module_syntax(): + """ + Return module syntax (Lua, Tcl) + """ + return ConfigurationVariables()['module_syntax'] + + def log_file_format(return_directory=False): """Return the format for the logfile or the directory""" idx = int(not return_directory) @@ -457,12 +487,12 @@ def set_tmpdir(tmpdir=None, raise_error=False): if tmpdir is not None: if not os.path.exists(tmpdir): os.makedirs(tmpdir) - current_tmpdir = tempfile.mkdtemp(prefix='easybuild-', dir=tmpdir) + current_tmpdir = tempfile.mkdtemp(prefix='eb-', dir=tmpdir) else: # use tempfile default parent dir - current_tmpdir = tempfile.mkdtemp(prefix='easybuild-') + current_tmpdir = tempfile.mkdtemp(prefix='eb-') except OSError, err: - _log.error("Failed to create temporary directory (tmpdir: %s): %s" % (tmpdir, err)) + raise EasyBuildError("Failed to create temporary directory (tmpdir: %s): %s", tmpdir, err) _log.info("Temporary directory used in this EasyBuild run: %s" % current_tmpdir) @@ -481,7 +511,7 @@ def set_tmpdir(tmpdir=None, raise_error=False): msg = "The temporary directory (%s) does not allow to execute files. " % tempfile.gettempdir() msg += "This can cause problems in the build process, consider using --tmpdir." if raise_error: - _log.error(msg) + raise EasyBuildError(msg) else: _log.warning(msg) else: @@ -489,6 +519,6 @@ def set_tmpdir(tmpdir=None, raise_error=False): os.remove(tmptest_file) except OSError, err: - _log.error("Failed to test whether temporary directory allows to execute files: %s" % err) + raise EasyBuildError("Failed to test whether temporary directory allows to execute files: %s", err) return current_tmpdir diff --git a/easybuild/tools/convert.py b/easybuild/tools/convert.py index b9b845bfe3..b4a2304597 100644 --- a/easybuild/tools/convert.py +++ b/easybuild/tools/convert.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2014 Ghent University +# Copyright 2014-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,6 +34,9 @@ from vsc.utils.missing import get_subclasses, nub from vsc.utils.wrapper import Wrapper +from easybuild.tools.build_log import EasyBuildError + + _log = fancylogger.getLogger('tools.convert', fname=False) @@ -56,7 +59,7 @@ def __init__(self, obj): if isinstance(obj, basestring): self.data = self._from_string(obj) else: - self.log.error('unsupported type %s for %s: %s' % (type(obj), self.__class__.__name__, obj)) + raise EasyBuildError("unsupported type %s for %s: %s", type(obj), self.__class__.__name__, obj) super(Convert, self).__init__(self.data) def _split_string(self, txt, sep=None, max=0): @@ -66,7 +69,7 @@ def _split_string(self, txt, sep=None, max=0): """ if sep is None: if self.SEPARATOR is None: - self.log.error('No SEPARATOR set, also no separator passed') + raise EasyBuildError("No SEPARATOR set, also no separator passed") else: sep = self.SEPARATOR return [x.strip() for x in re.split(r'' + sep, txt, maxsplit=max)] @@ -221,4 +224,4 @@ def get_convert_class(class_name): if len(res) == 1: return res[0] else: - _log.error('More then one Convert subclass found for name %s: %s' % (class_name, res)) + raise EasyBuildError("More than one Convert subclass found for name %s: %s", class_name, res) diff --git a/easybuild/tools/deprecated/__init__.py b/easybuild/tools/deprecated/__init__.py index 8c26fd6795..379c01f066 100644 --- a/easybuild/tools/deprecated/__init__.py +++ b/easybuild/tools/deprecated/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index c8033be791..628d6d5b2f 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/environment.py b/easybuild/tools/environment.py index 384c65febd..cd6b290d77 100644 --- a/easybuild/tools/environment.py +++ b/easybuild/tools/environment.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,10 +28,17 @@ @author: Toon Willems (Ghent University) @author: Ward Poelmans (Ghent University) """ +import copy import os from vsc.utils import fancylogger from vsc.utils.missing import shell_quote +from easybuild.tools.build_log import EasyBuildError + + +# take copy of original environemt, so we can restore (parts of) it later +ORIG_OS_ENVIRON = copy.deepcopy(os.environ) + _log = fancylogger.getLogger('environment', fname=False) @@ -53,7 +60,7 @@ def write_changes(filename): except IOError, err: if script is not None: script.close() - _log.error("Failed to write to %s: %s" % (filename, err)) + raise EasyBuildError("Failed to write to %s: %s", filename, err) reset_changes() @@ -71,15 +78,20 @@ def get_changes(): """ return _changes + def setvar(key, value): """ put key in the environment with value tracks added keys until write_changes has been called """ + if key in os.environ: + oldval_info = "previous value: '%s'" % os.environ[key] + else: + oldval_info = "previously undefined" # os.putenv() is not necessary. os.environ will call this. os.environ[key] = value _changes[key] = value - _log.info("Environment variable %s set to %s" % (key, value)) + _log.info("Environment variable %s set to %s (%s)", key, value, oldval_info) def unset_env_vars(keys): @@ -120,7 +132,7 @@ def read_environment(env_vars, strict=False): missing = ','.join(["%s / %s" % (k, v) for k, v in env_vars.items() if not k in result]) msg = 'Following name/variable not found in environment: %s' % missing if strict: - _log.error(msg) + raise EasyBuildError(msg) else: _log.debug(msg) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 67623e8169..f073381f48 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,6 +34,7 @@ @author: Ward Poelmans (Ghent University) @author: Fotis Georgatos (Uni.Lu, NTUA) """ +import glob import os import re import shutil @@ -44,7 +45,7 @@ from vsc.utils import fancylogger import easybuild.tools.environment as env -from easybuild.tools.build_log import print_msg # import build_log must stay, to activate use of EasyBuildLog +from easybuild.tools.build_log import EasyBuildError, print_msg # import build_log must stay, to use of EasyBuildLog from easybuild.tools.config import build_option from easybuild.tools import run @@ -147,7 +148,7 @@ def read_file(path, log_error=True): if f is not None: f.close() if log_error: - _log.error("Failed to read %s: %s" % (path, err)) + raise EasyBuildError("Failed to read %s: %s", path, err) else: return None @@ -168,7 +169,7 @@ def write_file(path, txt, append=False): # make sure file handle is always closed if f is not None: f.close() - _log.error("Failed to write to %s: %s" % (path, err)) + raise EasyBuildError("Failed to write to %s: %s", path, err) def remove_file(path): @@ -177,7 +178,7 @@ def remove_file(path): if os.path.exists(path): os.remove(path) except OSError, err: - _log.error("Failed to remove %s: %s", path, err) + raise EasyBuildError("Failed to remove %s: %s", path, err) def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False): @@ -186,19 +187,19 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False): - returns the directory name in case of success """ if not os.path.isfile(fn): - _log.error("Can't extract file %s: no such file" % fn) + raise EasyBuildError("Can't extract file %s: no such file", fn) mkdir(dest, parents=True) # use absolute pathnames from now on - absDest = os.path.abspath(dest) + abs_dest = os.path.abspath(dest) # change working directory try: - _log.debug("Unpacking %s in directory %s." % (fn, absDest)) - os.chdir(absDest) + _log.debug("Unpacking %s in directory %s.", fn, abs_dest) + os.chdir(abs_dest) except OSError, err: - _log.error("Can't change to directory %s: %s" % (absDest, err)) + raise EasyBuildError("Can't change to directory %s: %s", abs_dest, err) if not cmd: cmd = extract_cmd(fn, overwrite=overwrite) @@ -206,7 +207,7 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False): # complete command template with filename cmd = cmd % fn if not cmd: - _log.error("Can't extract file %s with unknown filetype" % fn) + raise EasyBuildError("Can't extract file %s with unknown filetype", fn) if extra_options: cmd = "%s %s" % (cmd, extra_options) @@ -232,7 +233,7 @@ def which(cmd): def det_common_path_prefix(paths): """Determine common path prefix for a given list of paths.""" if not isinstance(paths, list): - _log.error("det_common_path_prefix: argument must be of type list (got %s: %s)" % (type(paths), paths)) + raise EasyBuildError("det_common_path_prefix: argument must be of type list (got %s: %s)", type(paths), paths) elif not paths: return None @@ -290,7 +291,7 @@ def download_file(filename, url, path): _log.warning("IOError occurred while trying to download %s to %s: %s" % (url, path, err)) attempt_cnt += 1 except Exception, err: - _log.error("Unexpected error occurred when trying to download %s to %s: %s" % (url, path, err)) + raise EasyBuildError("Unexpected error occurred when trying to download %s to %s: %s", url, path, err) if not downloaded and attempt_cnt < max_attempts: _log.info("Attempt %d of downloading %s to %s failed, trying again..." % (attempt_cnt, url, path)) @@ -338,7 +339,11 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False): if ignore_dirs is None: ignore_dirs = ['.git', '.svn'] if not isinstance(ignore_dirs, list): - _log.error("search_file: ignore_dirs (%s) should be of type list, not %s" % (ignore_dirs, type(ignore_dirs))) + raise EasyBuildError("search_file: ignore_dirs (%s) should be of type list, not %s", + ignore_dirs, type(ignore_dirs)) + + # compile regex, case-insensitive + query = re.compile(query, re.I) var_lines = [] hit_lines = [] @@ -347,18 +352,16 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False): for path in paths: hits = [] hit_in_path = False - print_msg("Searching (case-insensitive) for '%s' in %s " % (query, path), log=_log, silent=silent) + print_msg("Searching (case-insensitive) for '%s' in %s " % (query.pattern, path), log=_log, silent=silent) - query = query.lower() for (dirpath, dirnames, filenames) in os.walk(path, topdown=True): for filename in filenames: - filename = os.path.join(dirpath, filename) - if filename.lower().find(query) != -1: + if query.search(filename): if not hit_in_path: var = "CFGS%d" % var_index var_index += 1 hit_in_path = True - hits.append(filename) + hits.append(os.path.join(dirpath, filename)) # do not consider (certain) hidden directories # note: we still need to consider e.g., .local ! @@ -366,6 +369,8 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False): # see http://stackoverflow.com/questions/13454164/os-walk-without-hidden-folders dirnames[:] = [d for d in dirnames if not d in ignore_dirs] + hits = sorted(hits) + if hits: common_prefix = det_common_path_prefix(hits) if short and common_prefix is not None and len(common_prefix) > len(var) * 2: @@ -386,12 +391,13 @@ def compute_checksum(path, checksum_type=DEFAULT_CHECKSUM): @param checksum_type: Type of checksum ('adler32', 'crc32', 'md5' (default), 'sha1', 'size') """ if not checksum_type in CHECKSUM_FUNCTIONS: - _log.error("Unknown checksum type (%s), supported types are: %s" % (checksum_type, CHECKSUM_FUNCTIONS.keys())) + raise EasyBuildError("Unknown checksum type (%s), supported types are: %s", + checksum_type, CHECKSUM_FUNCTIONS.keys()) try: checksum = CHECKSUM_FUNCTIONS[checksum_type](path) except IOError, err: - _log.error("Failed to read %s: %s" % (path, err)) + raise EasyBuildError("Failed to read %s: %s", path, err) except MemoryError, err: _log.warning("A memory error occured when computing the checksum for %s: %s" % (path, err)) checksum = 'dummy_checksum_due_to_memory_error' @@ -416,7 +422,7 @@ def calc_block_checksum(path, algorithm): algorithm.update(block) f.close() except IOError, err: - _log.error("Failed to read %s: %s" % (path, err)) + raise EasyBuildError("Failed to read %s: %s", path, err) return algorithm.hexdigest() @@ -443,7 +449,8 @@ def verify_checksum(path, checksums): elif isinstance(checksum, tuple) and len(checksum) == 2: typ, checksum = checksum else: - _log.error("Invalid checksum spec '%s', should be a string (MD5) or 2-tuple (type, value)." % checksum) + raise EasyBuildError("Invalid checksum spec '%s', should be a string (MD5) or 2-tuple (type, value).", + checksum) actual_checksum = compute_checksum(path, typ) _log.debug("Computed %s checksum for %s: %s (correct checksum: %s)" % (typ, path, actual_checksum, checksum)) @@ -482,7 +489,7 @@ def get_local_dirs_purged(): try: os.chdir(new_dir) except OSError, err: - _log.exception("Changing to dir %s from current dir %s failed: %s" % (new_dir, os.getcwd(), err)) + raise EasyBuildError("Changing to dir %s from current dir %s failed: %s", new_dir, os.getcwd(), err) lst = get_local_dirs_purged() # make sure it's a directory, and not a (single) file that was in a tarball for example @@ -548,7 +555,7 @@ def extract_cmd(filepath, overwrite=False): cmd_tmpl = "unzip -qq %(filepath)s" if cmd_tmpl is None: - _log.error('Unknown file type for file %s (%s)' % (filepath, exts)) + raise EasyBuildError('Unknown file type for file %s (%s)', filepath, exts) return cmd_tmpl % {'filepath': filepath, 'target': target} @@ -564,9 +571,9 @@ def det_patched_files(path=None, txt=None, omit_ab_prefix=False): txt = f.read() f.close() except IOError, err: - _log.error("Failed to read patch %s: %s" % (path, err)) + raise EasyBuildError("Failed to read patch %s: %s", path, err) elif txt is None: - _log.error("Either a file path or a string representing a patch should be supplied to det_patched_files") + raise EasyBuildError("Either a file path or a string representing a patch should be supplied") patched_files = [] for match in patched_regex.finditer(txt): @@ -611,15 +618,15 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None): """ if not os.path.isfile(patch_file): - _log.error("Can't find patch %s: no such file" % patch_file) + raise EasyBuildError("Can't find patch %s: no such file", patch_file) return if fn and not os.path.isfile(fn): - _log.error("Can't patch file %s: no such file" % fn) + raise EasyBuildError("Can't patch file %s: no such file", fn) return if not os.path.isdir(dest): - _log.error("Can't patch directory %s: no such directory" % dest) + raise EasyBuildError("Can't patch directory %s: no such directory", dest) return # copy missing files @@ -629,7 +636,7 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None): _log.debug("Copied patch %s to dir %s" % (patch_file, dest)) return 'ok' except IOError, err: - _log.error("Failed to copy %s to dir %s: %s" % (patch_file, dest, err)) + raise EasyBuildError("Failed to copy %s to dir %s: %s", patch_file, dest, err) return # use absolute paths @@ -644,14 +651,14 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None): patched_files = det_patched_files(path=apatch) if not patched_files: - _log.error("Can't guess patchlevel from patch %s: no testfile line found in patch" % apatch) + raise EasyBuildError("Can't guess patchlevel from patch %s: no testfile line found in patch", apatch) return patch_level = guess_patch_level(patched_files, adest) if patch_level is None: # patch_level can also be 0 (zero), so don't use "not patch_level" # no match - _log.error("Can't determine patch level for patch %s from directory %s" % (patch_file, adest)) + raise EasyBuildError("Can't determine patch level for patch %s from directory %s", patch_file, adest) else: _log.debug("Guessed patch level %d for patch %s" % (patch_level, patch_file)) @@ -663,13 +670,13 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None): os.chdir(adest) _log.debug("Changing to directory %s" % adest) except OSError, err: - _log.error("Can't change to directory %s: %s" % (adest, err)) + raise EasyBuildError("Can't change to directory %s: %s", adest, err) return patch_cmd = "patch -b -p%d -i %s" % (patch_level, apatch) result = run.run_cmd(patch_cmd, simple=True) if not result: - _log.error("Patching with patch %s failed" % patch_file) + raise EasyBuildError("Patching with patch %s failed", patch_file) return return result @@ -761,14 +768,14 @@ def adjust_permissions(name, permissionBits, add=True, onlyfiles=False, onlydirs failed_paths.append(path) if failed_paths: - _log.error("Failed to chmod/chown several paths: %s (last error: %s)" % (failed_paths, err)) + raise EasyBuildError("Failed to chmod/chown several paths: %s (last error: %s)", failed_paths, err) # we ignore some errors, but if there are to many, something is definitely wrong fail_ratio = fail_cnt / float(len(allpaths)) max_fail_ratio = 0.5 if fail_ratio > max_fail_ratio: - _log.error("%.2f%% of permissions/owner operations failed (more than %.2f%%), something must be wrong..." % - (100 * fail_ratio, 100 * max_fail_ratio)) + raise EasyBuildError("%.2f%% of permissions/owner operations failed (more than %.2f%%), " + "something must be wrong...", 100 * fail_ratio, 100 * max_fail_ratio) elif fail_cnt > 0: _log.debug("%.2f%% of permissions/owner operations failed, ignoring that..." % (100 * fail_ratio)) @@ -813,8 +820,7 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): # exit early if path already exists if not os.path.exists(path): - tup = (path, parents, set_gid, sticky) - _log.info("Creating directory %s (parents: %s, set_gid: %s, sticky: %s)" % tup) + _log.info("Creating directory %s (parents: %s, set_gid: %s, sticky: %s)", path, parents, set_gid, sticky) # set_gid and sticky bits are only set on new directories, so we need to determine the existing parent path existing_parent_path = os.path.dirname(path) try: @@ -826,7 +832,7 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): else: os.mkdir(path) except OSError, err: - _log.error("Failed to create directory %s: %s" % (path, err)) + raise EasyBuildError("Failed to create directory %s: %s", path, err) # set group ID and sticky bits, if desired bits = 0 @@ -840,7 +846,7 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): new_path = os.path.join(existing_parent_path, new_subdir.split(os.path.sep)[0]) adjust_permissions(new_path, bits, add=True, relative=True, recursive=True, onlydirs=True) except OSError, err: - _log.error("Failed to set groud ID/sticky bit: %s" % err) + raise EasyBuildError("Failed to set groud ID/sticky bit: %s", err) else: _log.debug("Not creating existing path %s" % path) @@ -868,19 +874,57 @@ def rmtree2(path, n=3): _log.debug("Failed to remove path %s with shutil.rmtree at attempt %d: %s" % (path, n, err)) time.sleep(2) if not ok: - _log.error("Failed to remove path %s with shutil.rmtree, even after %d attempts." % (path, n)) + raise EasyBuildError("Failed to remove path %s with shutil.rmtree, even after %d attempts.", path, n) else: _log.info("Path %s successfully removed." % path) +def move_logs(src_logfile, target_logfile): + """Move log file(s).""" + mkdir(os.path.dirname(target_logfile), parents=True) + src_logfile_len = len(src_logfile) + try: + + # there may be multiple log files, due to log rotation + app_logs = glob.glob('%s*' % src_logfile) + for app_log in app_logs: + # retain possible suffix + new_log_path = target_logfile + app_log[src_logfile_len:] + + # retain old logs + if os.path.exists(new_log_path): + i = 0 + oldlog_backup = "%s_%d" % (new_log_path, i) + while os.path.exists(oldlog_backup): + i += 1 + oldlog_backup = "%s_%d" % (new_log_path, i) + shutil.move(new_log_path, oldlog_backup) + _log.info("Moved existing log file %s to %s" % (new_log_path, oldlog_backup)) + + # move log to target path + shutil.move(app_log, new_log_path) + _log.info("Moved log file %s to %s" % (src_logfile, new_log_path)) + + except (IOError, OSError), err: + raise EasyBuildError("Failed to move log file(s) %s* to new log file %s*: %s" , + src_logfile, target_logfile, err) + + def cleanup(logfile, tempdir, testing): """Cleanup the specified log file and the tmp directory""" if not testing and logfile is not None: - os.remove(logfile) - print_msg('temporary log file %s has been removed.' % (logfile), log=None, silent=testing) + try: + for log in glob.glob('%s*' % logfile): + os.remove(log) + except OSError, err: + raise EasyBuildError("Failed to remove log file(s) %s*: %s", logfile, err) + print_msg('temporary log file(s) %s* have been removed.' % (logfile), log=None, silent=testing) if not testing and tempdir is not None: - shutil.rmtree(tempdir, ignore_errors=True) + try: + shutil.rmtree(tempdir, ignore_errors=True) + except OSError, err: + raise EasyBuildError("Failed to remove temporary directory %s: %s", tempdir, err) print_msg('temporary directory %s has been removed.' % (tempdir), log=None, silent=testing) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 1ff47d1555..cad7be7a8d 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,6 +37,8 @@ from vsc.utils import fancylogger from vsc.utils.patterns import Singleton +from easybuild.tools.build_log import EasyBuildError + _log = fancylogger.getLogger('github', fname=False) @@ -140,7 +142,7 @@ def listdir(self, path): return listing[1] else: self.log.warning("error: %s" % str(listing)) - self.log.exception("Invalid response from github (I/O error)") + raise EasyBuildError("Invalid response from github (I/O error)") def walk(self, top=None, topdown=True): """ @@ -205,15 +207,15 @@ def download(url, path=None): _, httpmsg = urllib.urlretrieve(url, path) _log.debug("Downloaded %s to %s" % (url, path)) except IOError, err: - _log.error("Failed to download %s to %s: %s" % (url, path, err)) + raise EasyBuildError("Failed to download %s to %s: %s", url, path, err) if not httpmsg.type == 'text/plain': - _log.error("Unexpected file type for %s: %s" % (path, httpmsg.type)) + raise EasyBuildError("Unexpected file type for %s: %s", path, httpmsg.type) else: try: return urllib2.urlopen(url).read() except urllib2.URLError, err: - _log.error("Failed to open %s for reading: %s" % (url, err)) + raise EasyBuildError("Failed to open %s for reading: %s", url, err) # a GitHub token is optional here, but can be used if available in order to be less susceptible to rate limiting github_token = fetch_github_token(github_user) @@ -235,14 +237,14 @@ def download(url, path=None): status, pr_data = 0, None _log.debug("status: %d, data: %s" % (status, pr_data)) if not status == HTTP_STATUS_OK: - tup = (pr, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, status, pr_data) - _log.error("Failed to get data for PR #%d from %s/%s (status: %d %s)" % tup) + raise EasyBuildError("Failed to get data for PR #%d from %s/%s (status: %d %s)", + pr, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, status, pr_data) # 'clean' on successful (or missing) test, 'unstable' on failed tests stable = pr_data['mergeable_state'] == GITHUB_MERGEABLE_STATE_CLEAN if not stable: - tup = (pr, GITHUB_MERGEABLE_STATE_CLEAN, pr_data['mergeable_state']) - _log.warning("Mergeable state for PR #%d is not '%s': %s." % tup) + _log.warning("Mergeable state for PR #%d is not '%s': %s.", + pr, GITHUB_MERGEABLE_STATE_CLEAN, pr_data['mergeable_state']) for key, val in sorted(pr_data.items()): _log.debug("\n%s:\n\n%s\n" % (key, val)) @@ -256,7 +258,7 @@ def download(url, path=None): # obtain last commit # get all commits, increase to (max of) 100 per page if pr_data['commits'] > GITHUB_MAX_PER_PAGE: - _log.error("PR #%s contains more than %s commits, can't obtain last commit" % (pr, GITHUB_MAX_PER_PAGE)) + raise EasyBuildError("PR #%s contains more than %s commits, can't obtain last commit", pr, GITHUB_MAX_PER_PAGE) status, commits_data = pr_url.commits.get(per_page=GITHUB_MAX_PER_PAGE) last_commit = commits_data[-1] _log.debug("Commits: %s, last commit: %s" % (commits_data, last_commit['sha'])) @@ -272,7 +274,7 @@ def download(url, path=None): all_files = [os.path.basename(x) for x in patched_files] tmp_files = os.listdir(path) if not sorted(tmp_files) == sorted(all_files): - _log.error("Not all patched files were downloaded to %s: %s vs %s" % (path, tmp_files, all_files)) + raise EasyBuildError("Not all patched files were downloaded to %s: %s vs %s", path, tmp_files, all_files) ec_files = [os.path.join(path, fn) for fn in tmp_files] @@ -298,7 +300,7 @@ def create_gist(txt, fn, descr=None, github_user=None): status, data = g.gists.post(body=body) if not status == HTTP_STATUS_CREATED: - _log.error("Failed to create gist; status %s, data: %s" % (status, data)) + raise EasyBuildError("Failed to create gist; status %s, data: %s", status, data) return data['html_url'] @@ -309,7 +311,7 @@ def post_comment_in_issue(issue, txt, repo=GITHUB_EASYCONFIGS_REPO, github_user= try: issue = int(issue) except ValueError, err: - _log.error("Failed to parse specified pull request number '%s' as an int: %s; " % (issue, err)) + raise EasyBuildError("Failed to parse specified pull request number '%s' as an int: %s; ", issue, err) github_token = fetch_github_token(github_user) g = RestClient(GITHUB_API_URL, username=github_user, token=github_token) @@ -317,7 +319,7 @@ def post_comment_in_issue(issue, txt, repo=GITHUB_EASYCONFIGS_REPO, github_user= status, data = pr_url.comments.post(body={'body': txt}) if not status == HTTP_STATUS_CREATED: - _log.error("Failed to create comment in PR %s#%d; status %s, data: %s" % (repo, issue, status, data)) + raise EasyBuildError("Failed to create comment in PR %s#%d; status %s, data: %s", repo, issue, status, data) class GithubToken(object): diff --git a/easybuild/tools/jenkins.py b/easybuild/tools/jenkins.py index 625984f0b5..0cc0ee4d1a 100644 --- a/easybuild/tools/jenkins.py +++ b/easybuild/tools/jenkins.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,6 +34,7 @@ from datetime import datetime from vsc.utils import fancylogger +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.version import FRAMEWORK_VERSION, EASYBLOCKS_VERSION @@ -108,7 +109,7 @@ def create_success(name, stats): root.writexml(output_file) output_file.close() except IOError, err: - _log.error("Failed to write out XML file %s: %s" % (filename, err)) + raise EasyBuildError("Failed to write out XML file %s: %s", filename, err) def aggregate_xml_in_dirs(base_dir, output_filename): @@ -149,7 +150,7 @@ def aggregate_xml_in_dirs(base_dir, output_filename): try: dom = xml.parse(xml_file) except IOError, err: - _log.error("Failed to read/parse XML file %s: %s" % (xml_file, err)) + raise EasyBuildError("Failed to read/parse XML file %s: %s", xml_file, err) # only one should be present, we are just discarding the rest testcase = dom.getElementsByTagName("testcase")[0] root.firstChild.appendChild(testcase) @@ -165,6 +166,6 @@ def aggregate_xml_in_dirs(base_dir, output_filename): root.writexml(output_file, addindent="\t", newl="\n") output_file.close() except IOError, err: - _log.error("Failed to write out XML file %s: %s" % (output_filename, err)) + raise EasyBuildError("Failed to write out XML file %s: %s", output_filename, err) print "Aggregate regtest results written to %s" % output_filename diff --git a/easybuild/tools/job/pbs_python.py b/easybuild/tools/job/pbs_python.py index 0a10b3f572..15323d4f5b 100644 --- a/easybuild/tools/job/pbs_python.py +++ b/easybuild/tools/job/pbs_python.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -35,6 +35,7 @@ import time from vsc.utils import fancylogger +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.job import JobServer @@ -174,8 +175,7 @@ def __init__(self, server, script, name, env_vars=None, try: self.pbsconn = self._server.connect_to_server() except Exception, err: - self.log.error("Failed to connect to PBS server: %s" % err) - raise + raise EasyBuildError("Failed to connect to the default pbs server: %s", err) # setup the resources requested @@ -294,7 +294,7 @@ def _submit(self): jobid = pbs.pbs_submit(self.pbsconn, pbs_attributes, scriptfn, self.queue, NULL) is_error, errormsg = pbs.error() if is_error or jobid is None: - self.log.error("Failed to submit job script %s (job id: %s, error %s)" % (scriptfn, jobid, errormsg)) + raise EasyBuildError("Failed to submit job script %s (job id: %s, error %s)", scriptfn, jobid, errormsg) else: self.log.debug("Succesful job submission returned jobid %s" % jobid) self.jobid = jobid @@ -309,13 +309,13 @@ def set_hold(self, hold_type=None): # only set hold if it wasn't set before if hold_type not in self.holds: if hold_type not in KNOWN_HOLD_TYPES: - self.log.error("set_hold: unknown hold type: %s (supported: %s)" % (hold_type, KNOWN_HOLD_TYPES)) + raise EasyBuildError("set_hold: unknown hold type: %s (supported: %s)", hold_type, KNOWN_HOLD_TYPES) # set hold, check for errors, and keep track of this hold ec = pbs.pbs_holdjob(self.pbsconn, self.jobid, hold_type, NULL) is_error, errormsg = pbs.error() if is_error or ec: - tup = (hold_type, self.jobid, is_error, ec, errormsg) - self.log.error("Failed to set hold of type %s on job %s (is_error: %s, exit code: %s, msg: %s)" % tup) + raise EasyBuildError("Failed to set hold of type %s on job %s (is_error: %s, exit code: %s, msg: %s)", + hold_type, self.jobid, is_error, ec, errormsg) else: self.holds.append(hold_type) else: @@ -330,14 +330,14 @@ def release_hold(self, hold_type=None): # only release hold if it was set if hold_type in self.holds: if hold_type not in KNOWN_HOLD_TYPES: - self.log.error("release_hold: unknown hold type: %s (supported: %s)" % (hold_type, KNOWN_HOLD_TYPES)) + raise EasyBuildError("release_hold: unknown hold type: %s (supported: %s)", hold_type, KNOWN_HOLD_TYPES) # release hold, check for errors, remove from list of holds ec = pbs.pbs_rlsjob(self.pbsconn, self.jobid, hold_type, NULL) self.log.debug("Released hold of type %s for job %s" % (hold_type, self.jobid)) is_error, errormsg = pbs.error() if is_error or ec: - tup = (hold_type, self.jobid, is_error, ec, errormsg) - self.log.error("Failed to release hold type %s on job %s (is_error: %s, exit code: %s, msg: %s)" % tup) + raise EasyBuildError("Failed to release hold type %s on job %s (is_error: %s, exit code: %s, msg: %s)", + hold_type, self.jobid, is_error, ec, errormsg) else: self.holds.remove(hold_type) else: @@ -418,7 +418,7 @@ def info(self, types=None): elif len(jobs) == 1: self.log.debug("Request for jobid %s returned one result %s" % (self.jobid, jobs)) else: - self.log.error("Request for jobid %s returned more then one result %s" % (self.jobid, jobs)) + raise EasyBuildError("Request for jobid %s returned more then one result %s", self.jobid, jobs) # only expect to have a list with one element j = jobs[0] @@ -433,6 +433,6 @@ def remove(self): """Remove the job with id jobid""" result = pbs.pbs_deljob(self.pbsconn, self.jobid, '') # use empty string, not NULL if result: - self.log.error("Failed to delete job %s: error %s" % (self.jobid, result)) + raise EasyBuildError("Failed to delete job %s: error %s", self.jobid, result) else: self.log.debug("Succesfully deleted job %s" % self.jobid) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index ee8046caba..d0ae5c2df0 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,14 +34,15 @@ """ import os import re +import sys import tempfile from vsc.utils import fancylogger +from vsc.utils.missing import get_subclasses -from easybuild.framework.easyconfig.easyconfig import ActiveMNS -from easybuild.tools import config -from easybuild.tools.config import build_option -from easybuild.tools.filetools import mkdir -from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import build_option, get_module_syntax, install_path +from easybuild.tools.filetools import mkdir, read_file +from easybuild.tools.modules import modules_tool from easybuild.tools.utilities import quote_str @@ -52,28 +53,34 @@ class ModuleGenerator(object): """ Class for generating module files. """ + SYNTAX = None # chars we want to escape in the generated modulefiles - CHARS_TO_ESCAPE = ["$"] + CHARS_TO_ESCAPE = None + + MODULE_FILE_EXTENSION = None + MODULE_HEADER = None def __init__(self, application, fake=False): + """ModuleGenerator constructor.""" self.app = application self.fake = fake self.tmpdir = None self.filename = None self.class_mod_file = None self.module_path = None + self.class_mod_files = None + self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) - def prepare(self): + def prepare(self, mod_symlink_paths): """ Creates the absolute filename for the module. """ mod_path_suffix = build_option('suffix_modules_path') - full_mod_name = self.app.full_mod_name + full_mod_name = self.app.full_mod_name + self.MODULE_FILE_EXTENSION # module file goes in general moduleclass category self.filename = os.path.join(self.module_path, mod_path_suffix, full_mod_name) # make symlink in moduleclass category - mod_symlink_paths = ActiveMNS().det_module_symlink_paths(self.app.cfg) self.class_mod_files = [os.path.join(self.module_path, p, full_mod_name) for p in mod_symlink_paths] # create directories and links @@ -95,7 +102,60 @@ def create_symlinks(self): os.remove(class_mod_file) os.symlink(self.filename, class_mod_file) except OSError, err: - _log.error("Failed to create symlinks from %s to %s: %s" % (self.class_mod_files, self.filename, err)) + raise EasyBuildError("Failed to create symlinks from %s to %s: %s", + self.class_mod_files, self.filename, err) + + def is_fake(self): + """Return whether this ModuleGenerator instance generates fake modules or not.""" + return self.fake + + def set_fake(self, fake): + """Determine whether this ModuleGenerator instance should generate fake modules.""" + self.log.debug("Updating fake for this ModuleGenerator instance to %s (was %s)" % (fake, self.fake)) + self.fake = fake + # fake mode: set installpath to temporary dir + if self.fake: + self.tmpdir = tempfile.mkdtemp() + self.log.debug("Fake mode: using %s (instead of %s)" % (self.tmpdir, self.module_path)) + self.module_path = self.tmpdir + else: + self.module_path = install_path('mod') + + def comment(self, str): + """Return given string formatted as a comment.""" + raise NotImplementedError + + def conditional_statement(self, condition, body, negative=False): + """Return formatted conditional statement, with given condition and body.""" + raise NotImplementedError + + +class ModuleGeneratorTcl(ModuleGenerator): + """ + Class for generating Tcl module files. + """ + SYNTAX = 'Tcl' + MODULE_FILE_EXTENSION = '' # no suffix for Tcl module files + MODULE_HEADER = '#%Module' + CHARS_TO_ESCAPE = ['$'] + + LOAD_REGEX = r"^\s*module\s+load\s+(\S+)" + LOAD_TEMPLATE = "module load %(mod_name)s" + + def comment(self, msg): + """Return string containing given message as a comment.""" + return "# %s\n" % msg + + def conditional_statement(self, condition, body, negative=False): + """Return formatted conditional statement, with given condition and body.""" + if negative: + lines = ["if { ![ %s ] } {" % condition] + else: + lines = ["if { [ %s ] } {" % condition] + + lines.append(' ' + body) + lines.extend(['}', '']) + return '\n'.join(lines) def get_description(self, conflict=True): """ @@ -104,28 +164,20 @@ def get_description(self, conflict=True): description = "%s - Homepage: %s" % (self.app.cfg['description'], self.app.cfg['homepage']) lines = [ - "#%%Module", # double % to escape string formatting! - "", + self.MODULE_HEADER.replace('%', '%%'), "proc ModulesHelp { } {", - " puts stderr { %(description)s", + " puts stderr { %(description)s", " }", - "}", - "", + '}', + '', "module-whatis {Description: %(description)s}", - "", - "set root %(installdir)s", - "", + '', + "set root %(installdir)s", ] if self.app.cfg['moduleloadnoconflict']: - lines.extend([ - "if { ![is-loaded %(name)s/%(version)s] } {", - " if { [is-loaded %(name)s] } {", - " module unload %(name)s", - " }", - "}", - "", - ]) + cond_unload = self.conditional_statement("is-loaded %(name)s", "module unload %(name)s") + lines.extend(['', self.conditional_statement("is-loaded %(name)s/%(version)s", cond_unload, negative=True)]) elif conflict: # conflict on 'name' part of module name (excluding version part at the end) @@ -133,9 +185,9 @@ def get_description(self, conflict=True): # - 'conflict GCC' for 'GCC/4.8.3' # - 'conflict Core/GCC' for 'Core/GCC/4.8.2' # - 'conflict Compiler/GCC/4.8.2/OpenMPI' for 'Compiler/GCC/4.8.2/OpenMPI/1.6.4' - lines.append("conflict %s\n" % os.path.dirname(self.app.short_mod_name)) + lines.extend(['', "conflict %s" % os.path.dirname(self.app.short_mod_name)]) - txt = '\n'.join(lines) % { + txt = '\n'.join(lines + ['']) % { 'name': self.app.name, 'version': self.app.version, 'description': description, @@ -152,26 +204,17 @@ def load_module(self, mod_name): # not wrapping the 'module load' with an is-loaded guard ensures recursive unloading; # when "module unload" is called on the module in which the depedency "module load" is present, # it will get translated to "module unload" - load_statement = ["module load %(mod_name)s"] + load_statement = [self.LOAD_TEMPLATE, ''] else: - load_statement = [ - "if { ![is-loaded %(mod_name)s] } {", - " module load %(mod_name)s", - "}", - ] - return '\n'.join([""] + load_statement + [""]) % {'mod_name': mod_name} + load_statement = [self.conditional_statement("is-loaded %(mod_name)s", self.LOAD_TEMPLATE, negative=True)] + return '\n'.join([''] + load_statement) % {'mod_name': mod_name} def unload_module(self, mod_name): """ Generate unload statements for module. """ - return '\n'.join([ - "", - "if { [is-loaded %(mod_name)s] } {", - " module unload %(mod_name)s", - "}", - "", - ]) % {'mod_name': mod_name} + cond_unload = self.conditional_statement("is-loaded %(mod)s", "module unload %(mod)s") % {'mod': mod_name} + return '\n'.join(['', cond_unload]) def prepend_paths(self, key, paths, allow_abs=False): """ @@ -180,16 +223,19 @@ def prepend_paths(self, key, paths, allow_abs=False): template = "prepend-path\t%s\t\t%s\n" if isinstance(paths, basestring): - _log.info("Wrapping %s into a list before using it to prepend path %s" % (paths, key)) + self.log.debug("Wrapping %s into a list before using it to prepend path %s" % (paths, key)) paths = [paths] - # make sure only relative paths are passed - for i in xrange(len(paths)): - if os.path.isabs(paths[i]) and not allow_abs: - _log.error("Absolute path %s passed to prepend_paths which only expects relative paths." % paths[i]) - elif not os.path.isabs(paths[i]): - # prepend $root (= installdir) for relative paths - paths[i] = "$root/%s" % paths[i] + for i, path in enumerate(paths): + if os.path.isabs(path) and not allow_abs: + raise EasyBuildError("Absolute path %s passed to prepend_paths which only expects relative paths.", + path) + elif not os.path.isabs(path): + # prepend $root (= installdir) for (non-empty) relative paths + if path: + paths[i] = os.path.join('$root', path) + else: + paths[i] = '$root' statements = [template % (key, p) for p in paths] return ''.join(statements) @@ -200,36 +246,31 @@ def use(self, paths): """ use_statements = [] for path in paths: - use_statements.append("module use %s" % path) - return '\n'.join(use_statements) + use_statements.append("module use %s\n" % path) + return ''.join(use_statements) - def set_environment(self, key, value): + def set_environment(self, key, value, relpath=False): """ Generate setenv statement for the given key/value pair. """ # quotes are needed, to ensure smooth working of EBDEVEL* modulefiles - return 'setenv\t%s\t\t%s\n' % (key, quote_str(value)) - + if relpath: + if value: + val = quote_str(os.path.join('$root', value)) + else: + val = '"$root"' + else: + val = quote_str(value) + return 'setenv\t%s\t\t%s\n' % (key, val) + def msg_on_load(self, msg): """ Add a message that should be printed when loading the module. """ # escape any (non-escaped) characters with special meaning by prefixing them with a backslash msg = re.sub(r'((? 0: + # recursively determine dependencies for these dependency modules, until depth is non-positive + moddeps = [dependencies_for(mod, depth=depth - 1) for mod in mods] + else: + # ignore any deeper dependencies + moddeps = [] + + # add dependencies of dependency modules only if they're not there yet + for moddepdeps in moddeps: + for dep in moddepdeps: + if not dep in mods: + mods.append(dep) + + return mods diff --git a/easybuild/tools/module_naming_scheme/__init__.py b/easybuild/tools/module_naming_scheme/__init__.py index 964cf582aa..084cf7f165 100644 --- a/easybuild/tools/module_naming_scheme/__init__.py +++ b/easybuild/tools/module_naming_scheme/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2014 Ghent University +# Copyright 2011-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/easybuild_mns.py b/easybuild/tools/module_naming_scheme/easybuild_mns.py index 864002431f..801ac6b4d5 100644 --- a/easybuild/tools/module_naming_scheme/easybuild_mns.py +++ b/easybuild/tools/module_naming_scheme/easybuild_mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/hierarchical_mns.py b/easybuild/tools/module_naming_scheme/hierarchical_mns.py index 587397ae21..7215d1c01b 100644 --- a/easybuild/tools/module_naming_scheme/hierarchical_mns.py +++ b/easybuild/tools/module_naming_scheme/hierarchical_mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -33,6 +33,7 @@ import re from vsc.utils import fancylogger +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.module_naming_scheme import ModuleNamingScheme from easybuild.tools.module_naming_scheme.toolchain import det_toolchain_compilers, det_toolchain_mpi @@ -102,9 +103,10 @@ def det_toolchain_compilers_name_version(self, tc_comps): tc_comp_ver = tc_comp_ver_tmpl % comp_versions # make sure that icc/ifort versions match if tc_comp_name == 'intel' and comp_versions['icc'] != comp_versions['ifort']: - self.log.error("Bumped into different versions for Intel compilers: %s" % comp_versions) + raise EasyBuildError("Bumped into different versions for Intel compilers: %s", comp_versions) else: - self.log.error("Unknown set of toolchain compilers, module naming scheme needs work: %s" % comp_names) + raise EasyBuildError("Unknown set of toolchain compilers, module naming scheme needs work: %s", + comp_names) res = (tc_comp_name, tc_comp_ver) return res @@ -180,10 +182,9 @@ def det_modpath_extensions(self, ec): elif modclass == MODULECLASS_MPI: if tc_comp_info is None: - tup = (ec['toolchain'], ec['name'], ec['version']) - error_msg = ("No compiler available in toolchain %s used to install MPI library %s v%s, " - "which is required by the active module naming scheme.") % tup - self.log.error(error_msg) + raise EasyBuildError("No compiler available in toolchain %s used to install MPI library %s v%s, " + "which is required by the active module naming scheme.", + ec['toolchain'], ec['name'], ec['version']) else: tc_comp_name, tc_comp_ver = tc_comp_info fullver = self.det_full_version(ec) diff --git a/easybuild/tools/module_naming_scheme/mns.py b/easybuild/tools/module_naming_scheme/mns.py index ce6de596c1..670493fb78 100644 --- a/easybuild/tools/module_naming_scheme/mns.py +++ b/easybuild/tools/module_naming_scheme/mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2014 Ghent University +# Copyright 2011-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -32,6 +32,8 @@ from vsc.utils import fancylogger from vsc.utils.patterns import Singleton +from easybuild.tools.build_log import EasyBuildError + class ModuleNamingScheme(object): """Abstract class for a module naming scheme implementation.""" @@ -50,7 +52,8 @@ def is_sufficient(self, keys): if self.REQUIRED_KEYS is not None: return set(keys).issuperset(set(self.REQUIRED_KEYS)) else: - self.log.error("Constant REQUIRED_KEYS is not defined, should specify required easyconfig parameters.") + raise EasyBuildError("Constant REQUIRED_KEYS is not defined, " + "should specify required easyconfig parameters.") def requires_toolchain_details(self): """ @@ -81,6 +84,18 @@ def det_short_module_name(self, ec): # by default: full module name doesn't include a $MODULEPATH subdir return self.det_full_module_name(ec) + def det_install_subdir(self, ec): + """ + Determine name of software installation subdirectory of install path. + + @param ec: dict-like object with easyconfig parameter values; for now only the 'name', + 'version', 'versionsuffix' and 'toolchain' parameters are guaranteed to be available + + @return: string with name of subdirectory, e.g.: '///' + """ + # by default: use full module name as name for install subdir + return self.det_full_module_name(ec) + def det_module_subdir(self, ec): """ Determine subdirectory for module file in $MODULEPATH. @@ -134,7 +149,7 @@ def is_short_modname_for(self, short_modname, name): modname_regex = re.compile('^%s/\S+$' % re.escape(name)) res = bool(modname_regex.match(short_modname)) - tup = (short_modname, name, modname_regex.pattern, res) - self.log.debug("Checking whether '%s' is a module name for software with name '%s' via regex %s: %s" % tup) + self.log.debug("Checking whether '%s' is a module name for software with name '%s' via regex %s: %s", + short_modname, name, modname_regex.pattern, res) return res diff --git a/easybuild/tools/module_naming_scheme/toolchain.py b/easybuild/tools/module_naming_scheme/toolchain.py index d835fcf2a9..5c63be336a 100644 --- a/easybuild/tools/module_naming_scheme/toolchain.py +++ b/easybuild/tools/module_naming_scheme/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2014 Ghent University +# Copyright 2014-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -30,6 +30,7 @@ from vsc.utils import fancylogger from easybuild.framework.easyconfig.easyconfig import process_easyconfig, robot_find_easyconfig +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME @@ -77,7 +78,7 @@ def det_toolchain_element_details(tc, elem): if tc_ec['name'] == elem: tc_elem_details = tc_ec else: - _log.error("No toolchain element '%s' found for toolchain %s: %s" % (elem, tc.as_dict(), tc_ec)) + raise EasyBuildError("No toolchain element '%s' found for toolchain %s: %s", elem, tc.as_dict(), tc_ec) _toolchain_details_cache[key] = tc_elem_details _log.debug("Obtained details for '%s' in toolchain '%s', added to cache" % (elem, tc_dict)) return _toolchain_details_cache[key] @@ -95,14 +96,15 @@ def det_toolchain_compilers(ec): tc_comps = None elif not TOOLCHAIN_COMPILER in tc_elems: # every toolchain should have at least a compiler - _log.error("No compiler found in toolchain %s: %s" % (ec.toolchain.as_dict(), tc_elems)) + raise EasyBuildError("No compiler found in toolchain %s: %s", ec.toolchain.as_dict(), tc_elems) elif tc_elems[TOOLCHAIN_COMPILER]: tc_comps = [] for comp_elem in tc_elems[TOOLCHAIN_COMPILER]: tc_comps.append(det_toolchain_element_details(ec.toolchain, comp_elem)) else: - _log.error("Empty list of compilers for %s toolchain definition: %s" % (ec.toolchain.as_dict(), tc_elems)) - _log.debug("Found compilers %s for toolchain %s (%s)" % (tc_comps, ec.toolchain.name, ec.toolchain.as_dict())) + raise EasyBuildError("Empty list of compilers for %s toolchain definition: %s", + ec.toolchain.as_dict(), tc_elems) + _log.debug("Found compilers %s for toolchain %s (%s)", tc_comps, ec.toolchain.name, ec.toolchain.as_dict()) return tc_comps @@ -116,7 +118,8 @@ def det_toolchain_mpi(ec): tc_elems = ec.toolchain.definition() if TOOLCHAIN_MPI in tc_elems: if not tc_elems[TOOLCHAIN_MPI]: - _log.error("Empty list of MPI libs for %s toolchain definition: %s" % (ec.toolchain.as_dict(), tc_elems)) + raise EasyBuildError("Empty list of MPI libs for %s toolchain definition: %s", + ec.toolchain.as_dict(), tc_elems) # assumption: only one MPI toolchain element tc_mpi = det_toolchain_element_details(ec.toolchain, tc_elems[TOOLCHAIN_MPI][0]) else: diff --git a/easybuild/tools/module_naming_scheme/utilities.py b/easybuild/tools/module_naming_scheme/utilities.py index 8bd5053231..da3711ad00 100644 --- a/easybuild/tools/module_naming_scheme/utilities.py +++ b/easybuild/tools/module_naming_scheme/utilities.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index b1e99c92fe..3700a1a8ab 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1,5 +1,5 @@ -# # -# Copyright 2009-2014 Ghent University +## +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -21,7 +21,7 @@ # # You should have received a copy of the GNU General Public License # along with EasyBuild. If not, see . -# # +## """ This python module implements the environment modules functionality: - loading modules @@ -38,7 +38,6 @@ import os import re import subprocess -import sys from distutils.version import StrictVersion from subprocess import PIPE from vsc.utils import fancylogger @@ -47,11 +46,10 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, get_modules_tool, install_path -from easybuild.tools.environment import restore_env +from easybuild.tools.environment import ORIG_OS_ENVIRON, restore_env from easybuild.tools.filetools import convert_name, mkdir, read_file, path_matches, which from easybuild.tools.module_naming_scheme import DEVEL_MODULE_SUFFIX from easybuild.tools.run import run_cmd -from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME, DUMMY_TOOLCHAIN_VERSION from vsc.utils.missing import nub # software root/version environment variable name prefixes @@ -59,9 +57,9 @@ VERSION_ENV_VAR_NAME_PREFIX = "EBVERSION" DEVEL_ENV_VAR_NAME_PREFIX = "EBDEVEL" -# keep track of original LD_LIBRARY_PATH, because we can change it by loading modules and break modulecmd +# environment variables to reset/restore when running a module command (to avoid breaking it) # see e.g., https://bugzilla.redhat.com/show_bug.cgi?id=719785 -LD_LIBRARY_PATH = os.getenv('LD_LIBRARY_PATH', '') +LD_ENV_VAR_KEYS = ['LD_LIBRARY_PATH', 'LD_PRELOAD'] output_matchers = { # matches whitespace and module-listing headers @@ -124,7 +122,8 @@ class ModulesTool(object): TERSE_OPTION = (0, '--terse') # module command to use COMMAND = None - # environment variable to determine the module command (instead of COMMAND) + # environment variable to determine path to module command; + # used as fallback in case command is not available in $PATH COMMAND_ENVIRONMENT = None # run module command explicitly using this shell COMMAND_SHELL = None @@ -158,12 +157,21 @@ def __init__(self, mod_paths=None, testing=False): # actual module command (i.e., not the 'module' wrapper function, but the binary) self.cmd = self.COMMAND - if self.COMMAND_ENVIRONMENT is not None and self.COMMAND_ENVIRONMENT in os.environ: - self.log.debug('Set command via environment variable %s' % self.COMMAND_ENVIRONMENT) - self.cmd = os.environ[self.COMMAND_ENVIRONMENT] + env_cmd_path = os.environ.get(self.COMMAND_ENVIRONMENT) + # only use command path in environment variable if command in not available in $PATH + if which(self.cmd) is None and env_cmd_path is not None: + self.log.debug('Set command via environment variable %s: %s', self.COMMAND_ENVIRONMENT, self.cmd) + self.cmd = env_cmd_path + + # check whether paths obtained via $PATH and $LMOD_CMD are different + elif which(self.cmd) != env_cmd_path: + self.log.debug("Different paths found for module command '%s' via which/$PATH and $%s: %s vs %s", + self.COMMAND, self.COMMAND_ENVIRONMENT, self.cmd, env_cmd_path) + + # make sure the module command was found if self.cmd is None: - self.log.error('No command set.') + raise EasyBuildError("No command set.") else: self.log.debug('Using command %s' % self.cmd) @@ -188,7 +196,7 @@ def modules(self): def set_and_check_version(self): """Get the module version, and check any requirements""" if self.VERSION_REGEXP is None: - self.log.error('No VERSION_REGEXP defined') + raise EasyBuildError("No VERSION_REGEXP defined") try: txt = self.run_module(self.VERSION_OPTION, return_output=True) @@ -207,16 +215,17 @@ def set_and_check_version(self): self.log.info("Converted actual version to '%s'" % self.version) else: - self.log.error("Failed to determine version from option '%s' output: %s" % (self.VERSION_OPTION, txt)) + raise EasyBuildError("Failed to determine version from option '%s' output: %s", + self.VERSION_OPTION, txt) except (OSError), err: - self.log.error("Failed to check version: %s" % err) + raise EasyBuildError("Failed to check version: %s", err) if self.REQ_VERSION is None: - self.log.debug('No version requirement defined.') + self.log.debug("No version requirement defined.") else: if StrictVersion(self.version) < StrictVersion(self.REQ_VERSION): - msg = "EasyBuild requires v%s >= v%s (no rc), found v%s" - self.log.error(msg % (self.__class__.__name__, self.REQ_VERSION, self.version)) + raise EasyBuildError("EasyBuild requires v%s >= v%s (no rc), found v%s", + self.__class__.__name__, self.REQ_VERSION, self.version) else: self.log.debug('Version %s matches requirement %s' % (self.version, self.REQ_VERSION)) @@ -228,7 +237,7 @@ def check_cmd_avail(self): self.log.info("Full path for module command is %s, so using it" % self.cmd) else: mod_tool = self.__class__.__name__ - self.log.error("%s modules tool can not be used, '%s' command is not available." % (mod_tool, self.cmd)) + raise EasyBuildError("%s modules tool can not be used, '%s' command is not available.", mod_tool, self.cmd) def check_module_function(self, allow_mismatch=False, regex=None): """Check whether selected module tool matches 'module' function definition.""" @@ -259,7 +268,7 @@ def check_module_function(self, allow_mismatch=False, regex=None): else: msg += "Or alternatively, use --allow-modules-tool-mismatch to stop treating this as an error. " msg += "Obtained definition of 'module' function: %s" % out - self.log.error(msg) + raise EasyBuildError(msg) else: # module function may not be defined (weird, but fine) self.log.warning("No 'module' function defined, can't check if it matches %s." % mod_details) @@ -374,14 +383,14 @@ def exists(self, mod_name): """NO LONGER SUPPORTED: use exist method instead""" self.log.nosupport("exists() is not supported anymore, use exist([]) instead", '2.0') - def load(self, modules, mod_paths=None, purge=False, orig_env=None): + def load(self, modules, mod_paths=None, purge=False, init_env=None): """ Load all requested modules. @param modules: list of modules to load @param mod_paths: list of module paths to activate before loading @param purge: whether or not a 'module purge' should be run before loading - @param orig_env: original environment to restore after running 'module purge' + @param init_env: original environment to restore after running 'module purge' """ if mod_paths is None: mod_paths = [] @@ -389,9 +398,9 @@ def load(self, modules, mod_paths=None, purge=False, orig_env=None): # purge all loaded modules if desired if purge: self.purge() - # restore original environment if provided - if orig_env is not None: - restore_env(orig_env) + # restore initial environment if provided + if init_env is not None: + restore_env(init_env) # make sure $MODULEPATH is set correctly after purging self.check_module_path() @@ -440,9 +449,10 @@ def get_value_from_modulefile(self, mod_name, regex): if res: return res.group(1) else: - self.log.error("Failed to determine value from 'show' (pattern: '%s') in %s" % (regex.pattern, modinfo)) + raise EasyBuildError("Failed to determine value from 'show' (pattern: '%s') in %s", + regex.pattern, modinfo) else: - raise EasyBuildError("Can't get value from a non-existing module %s" % mod_name) + raise EasyBuildError("Can't get value from a non-existing module %s", mod_name) def modulefile_path(self, mod_name): """Get the path of the module file for the specified module.""" @@ -451,9 +461,9 @@ def modulefile_path(self, mod_name): modpath_re = re.compile('^\s*(?P[^/\n]*/[^ ]+):$', re.M) return self.get_value_from_modulefile(mod_name, modpath_re) - def set_ld_library_path(self, ld_library_paths): - """Set $LD_LIBRARY_PATH to the given list of paths.""" - os.environ['LD_LIBRARY_PATH'] = ':'.join(ld_library_paths) + def set_path_env_var(self, key, paths): + """Set path environment variable to the given list of paths.""" + os.environ[key] = os.pathsep.join(paths) def run_module(self, *args, **kwargs): """ @@ -478,19 +488,19 @@ def run_module(self, *args, **kwargs): self.log.debug('Current MODULEPATH: %s' % os.environ.get('MODULEPATH', '')) - # change our LD_LIBRARY_PATH here + # restore selected original environment variables before running module command environ = os.environ.copy() - environ['LD_LIBRARY_PATH'] = LD_LIBRARY_PATH - cur_ld_library_path = os.environ.get('LD_LIBRARY_PATH', '') - new_ld_library_path = environ['LD_LIBRARY_PATH'] - self.log.debug("Adjusted LD_LIBRARY_PATH from '%s' to '%s'" % (cur_ld_library_path, new_ld_library_path)) + for key in LD_ENV_VAR_KEYS: + environ[key] = ORIG_OS_ENVIRON.get(key, '') + self.log.debug("Changing %s from '%s' to '%s' in environment for module command", + key, os.environ.get(key, ''), environ[key]) # prefix if a particular shell is specified, using shell argument to Popen doesn't work (no output produced (?)) cmdlist = [self.cmd, 'python'] if self.COMMAND_SHELL is not None: if not isinstance(self.COMMAND_SHELL, (list, tuple)): - msg = 'COMMAND_SHELL needs to be list or tuple, now %s (value %s)' - self.log.error(msg % (type(self.COMMAND_SHELL), self.COMMAND_SHELL)) + raise EasyBuildError("COMMAND_SHELL needs to be list or tuple, now %s (value %s)", + type(self.COMMAND_SHELL), self.COMMAND_SHELL) cmdlist = self.COMMAND_SHELL + cmdlist full_cmd = ' '.join(cmdlist + args) @@ -504,11 +514,12 @@ def run_module(self, *args, **kwargs): if kwargs.get('return_output', False): return stdout + stderr else: - # the module command was run with an outdated LD_LIBRARY_PATH, which will be adjusted on loading a module + # the module command was run with an outdated selected environment variables (see LD_ENV_VAR_KEYS list) + # which will be adjusted on loading a module; # this needs to be taken into account when updating the environment via produced output, see below - # keep track of current LD_LIBRARY_PATH, so we can correct the adjusted LD_LIBRARY_PATH below - prev_ld_library_path = os.environ.get('LD_LIBRARY_PATH', '').split(':')[::-1] + # keep track of current values of select env vars, so we can correct the adjusted values below + prev_ld_values = dict([(key, os.environ.get(key, '').split(os.pathsep)[::-1]) for key in LD_ENV_VAR_KEYS]) # Change the environment try: @@ -518,16 +529,16 @@ def run_module(self, *args, **kwargs): exec stdout except Exception, err: out = "stdout: %s, stderr: %s" % (stdout, stderr) - raise EasyBuildError("Changing environment as dictated by module failed: %s (%s)" % (err, out)) + raise EasyBuildError("Changing environment as dictated by module failed: %s (%s)", err, out) - # correct LD_LIBRARY_PATH as yielded by the adjustments made + # correct values of selected environment variables as yielded by the adjustments made # make sure we get the order right (reverse lists with [::-1]) - curr_ld_library_path = os.environ.get('LD_LIBRARY_PATH', '').split(':') - new_ld_library_path = [x for x in nub(prev_ld_library_path + curr_ld_library_path[::-1]) if len(x)][::-1] + for key in LD_ENV_VAR_KEYS: + curr_ld_val = os.environ.get(key, '').split(os.pathsep) + new_ld_val = [x for x in nub(prev_ld_values[key] + curr_ld_val[::-1]) if x][::-1] - self.log.debug("Correcting paths in LD_LIBRARY_PATH from %s to %s" % - (curr_ld_library_path, new_ld_library_path)) - self.set_ld_library_path(new_ld_library_path) + self.log.debug("Correcting paths in $%s from %s to %s" % (key, curr_ld_val, new_ld_val)) + self.set_path_env_var(key, new_ld_val) # Process stderr result = [] @@ -537,7 +548,6 @@ def run_module(self, *args, **kwargs): error = output_matchers['error'].search(line) if error: - self.log.error(line) raise EasyBuildError(line) modules = output_matchers['available'].finditer(line) @@ -568,30 +578,6 @@ def read_module_file(self, mod_name): return read_file(modfilepath) - def dependencies_for(self, mod_name, depth=sys.maxint): - """ - Obtain a list of dependencies for the given module, determined recursively, up to a specified depth (optionally) - @param depth: recursion depth (default is sys.maxint, which should be equivalent to infinite recursion depth) - """ - modtxt = self.read_module_file(mod_name) - loadregex = re.compile(r"^\s*module\s+load\s+(\S+)", re.M) - mods = loadregex.findall(modtxt) - - if depth > 0: - # recursively determine dependencies for these dependency modules, until depth is non-positive - moddeps = [self.dependencies_for(mod, depth=depth - 1) for mod in mods] - else: - # ignore any deeper dependencies - moddeps = [] - - # add dependencies of dependency modules only if they're not there yet - for moddepdeps in moddeps: - for dep in moddepdeps: - if not dep in mods: - mods.append(dep) - - return mods - def modpath_extensions_for(self, mod_names): """ Determine dictionary with $MODULEPATH extensions for specified modules. @@ -600,7 +586,7 @@ def modpath_extensions_for(self, mod_names): self.log.debug("Determining $MODULEPATH extensions for modules %s" % mod_names) # copy environment so we can restore it - orig_env = os.environ.copy() + env = os.environ.copy() modpath_exts = {} for mod_name in mod_names: @@ -616,8 +602,8 @@ def modpath_extensions_for(self, mod_names): # this is required to obtain the list of $MODULEPATH extensions they make (via 'module show') self.load([mod_name]) - # restore original environment (modules may have been loaded above) - restore_env(orig_env) + # restore environment (modules may have been loaded above) + restore_env(env) return modpath_exts @@ -653,7 +639,7 @@ def path_to_top_of_module_tree(self, top_paths, mod_name, full_mod_subdir, deps, @param modpath_exts: list of module path extensions for each of the dependency modules """ # copy environment so we can restore it - orig_env = os.environ.copy() + env = os.environ.copy() if path_matches(full_mod_subdir, top_paths): self.log.debug("Top of module tree reached with %s (module subdir: %s)" % (mod_name, full_mod_subdir)) @@ -678,8 +664,8 @@ def path_to_top_of_module_tree(self, top_paths, mod_name, full_mod_subdir, deps, full_mod_subdirs.append(dep_full_mod_subdir) mods_to_top.append(dep) - tup = (dep, dep_full_mod_subdir, full_modpath_exts) - self.log.debug("Found module to top of module tree: %s (subdir: %s, modpath extensions %s)" % tup) + self.log.debug("Found module to top of module tree: %s (subdir: %s, modpath extensions %s)", + dep, dep_full_mod_subdir, full_modpath_exts) if full_modpath_exts: # load module for this dependency, since it may extend $MODULEPATH to make dependencies available @@ -687,7 +673,7 @@ def path_to_top_of_module_tree(self, top_paths, mod_name, full_mod_subdir, deps, self.load([dep]) # restore original environment (modules may have been loaded above) - restore_env(orig_env) + restore_env(env) path = mods_to_top[:] if mods_to_top: @@ -732,11 +718,11 @@ class EnvironmentModulesTcl(EnvironmentModulesC): REQ_VERSION = None VERSION_REGEXP = r'^Modules\s+Release\s+Tcl\s+(?P\d\S*)\s' - def set_ld_library_path(self, ld_library_paths): - """Set $LD_LIBRARY_PATH to the given list of paths.""" - super(EnvironmentModulesTcl, self).set_ld_library_path(ld_library_paths) + def set_path_env_var(self, key, paths): + """Set environment variable with given name to the given list of paths.""" + super(EnvironmentModulesTcl, self).set_path_env_var(key, paths) # for Tcl environment modules, we need to make sure the _modshare env var is kept in sync - os.environ['LD_LIBRARY_PATH_modshare'] = ':1:'.join(ld_library_paths) + os.environ['%s_modshare' % key] = ':1:'.join(paths) def run_module(self, *args, **kwargs): """ @@ -845,7 +831,7 @@ def update(self): (stdout, stderr) = proc.communicate() if stderr: - self.log.error("An error occured when running '%s': %s" % (' '.join(cmd), stderr)) + raise EasyBuildError("An error occured when running '%s': %s", ' '.join(cmd), stderr) if self.testing: # don't actually update local cache when testing, just return the cache contents @@ -861,7 +847,7 @@ def update(self): cache_file.write(stdout) cache_file.close() except (IOError, OSError), err: - self.log.error("Failed to update Lmod spider cache %s: %s" % (cache_fp, err)) + raise EasyBuildError("Failed to update Lmod spider cache %s: %s", cache_fp, err) def prepend_module_path(self, path): # Lmod pushes a path to the front on 'module use' @@ -872,7 +858,7 @@ def prepend_module_path(self, path): def get_software_root_env_var_name(name): """Return name of environment variable for software root.""" newname = convert_name(name, upper=True) - return ''.join([ROOT_ENV_VAR_NAME_PREFIX, newname]) + return ROOT_ENV_VAR_NAME_PREFIX + newname def get_software_root(name, with_env_var=False): @@ -923,7 +909,8 @@ def get_software_libdir(name, only_one=True, fs=None): if len(res) == 1: res = res[0] else: - _log.error("Multiple library subdirectories found for %s in %s: %s" % (name, root, ', '.join(res))) + raise EasyBuildError("Multiple library subdirectories found for %s in %s: %s", + name, root, ', '.join(res)) return res else: # return None if software package root could not be determined @@ -933,7 +920,7 @@ def get_software_libdir(name, only_one=True, fs=None): def get_software_version_env_var_name(name): """Return name of environment variable for software root.""" newname = convert_name(name, upper=True) - return ''.join([VERSION_ENV_VAR_NAME_PREFIX, newname]) + return VERSION_ENV_VAR_NAME_PREFIX + newname def get_software_version(name): diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 5429603d37..5aa5e2cbf8 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -40,7 +40,7 @@ from distutils.version import LooseVersion from vsc.utils.missing import nub -from easybuild.framework.easyblock import EasyBlock +from easybuild.framework.easyblock import MODULE_ONLY_STEPS, SOURCE_STEP, EasyBlock from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR from easybuild.framework.easyconfig.constants import constant_documentation from easybuild.framework.easyconfig.format.pyheaderconfigobj import build_easyconfig_constants_dict @@ -49,18 +49,20 @@ from easybuild.framework.easyconfig.tools import get_paths_for from easybuild.framework.extension import Extension from easybuild.tools import build_log, config, run # @UnusedImport make sure config is always initialized! -from easybuild.tools.config import DEFAULT_LOGFILE_FORMAT -from easybuild.tools.config import DEFAULT_MNS, DEFAULT_MODULES_TOOL, DEFAULT_MODULECLASSES -from easybuild.tools.config import DEFAULT_PATH_SUBDIRS, DEFAULT_PREFIX, DEFAULT_REPOSITORY -from easybuild.tools.config import get_pretend_installpath -from easybuild.tools.config import mk_full_default_path +from easybuild.tools.build_log import EasyBuildError, raise_easybuilderror +from easybuild.tools.config import DEFAULT_LOGFILE_FORMAT, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL +from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PREFIX, DEFAULT_REPOSITORY +from easybuild.tools.config import DEFAULT_STRICT, get_pretend_installpath, mk_full_default_path from easybuild.tools.config import PREFERRED_JOB_BACKENDS +from easybuild.tools.configobj import ConfigObj, ConfigObjError from easybuild.tools.docs import FORMAT_RST, FORMAT_TXT, avail_easyconfig_params from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, fetch_github_token from easybuild.tools.job import avail_job_backends, preferred_job_backend from easybuild.tools.modules import avail_modules_tools +from easybuild.tools.module_generator import ModuleGeneratorLua, avail_module_generators from easybuild.tools.module_naming_scheme import GENERAL_CLASS from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes +from easybuild.tools.modules import Lmod from easybuild.tools.ordereddict import OrderedDict from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.repository.repository import avail_repositories @@ -82,18 +84,23 @@ class EasyBuildOptions(GeneralOption): VERSION = this_is_easybuild() DEFAULT_LOGLEVEL = 'INFO' - DEFAULT_CONFIGFILES = DEFAULT_SYS_CFGFILES + [DEFAULT_USER_CFGFILE] + DEFAULT_CONFIGFILES = DEFAULT_SYS_CFGFILES[:] + if os.path.exists(DEFAULT_USER_CFGFILE): + DEFAULT_CONFIGFILES.append(DEFAULT_USER_CFGFILE) ALLOPTSMANDATORY = False # allow more than one argument + CONFIGFILES_RAISE_MISSING = True # don't allow non-existing config files to be specified def __init__(self, *args, **kwargs): """Constructor.""" + self.default_repositorypath = [mk_full_default_path('repositorypath')] self.default_robot_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None) or [] # set up constants to seed into config files parser, by section self.go_cfg_constants = { self.DEFAULTSECT: { + 'DEFAULT_REPOSITORYPATH': (self.default_repositorypath[0], "Default easyconfigs repository path"), 'DEFAULT_ROBOT_PATHS': (os.pathsep.join(self.default_robot_paths), "List of default robot paths ('%s'-separated)" % os.pathsep), } @@ -125,11 +132,12 @@ def basic_options(self): 'robot': ("Enable dependency resolution, using easyconfigs in specified paths", 'pathlist', 'store_or_None', [], 'r', {'metavar': 'PATH[%sPATH]' % os.pathsep}), 'robot-paths': ("Additional paths to consider by robot for easyconfigs (--robot paths get priority)", - 'pathlist', 'store', self.default_robot_paths, {'metavar': 'PATH[%sPATH]' % os.pathsep}), + 'pathlist', 'add_flex', self.default_robot_paths, {'metavar': 'PATH[%sPATH]' % os.pathsep}), 'skip': ("Skip existing software (useful for installing additional packages)", None, 'store_true', False, 'k'), - 'stop': ("Stop the installation after certain step", 'choice', 'store_or_None', 'source', 's', all_stops), - 'strict': ("Set strictness level", 'choice', 'store', run.WARN, strictness_options), + 'stop': ("Stop the installation after certain step", + 'choice', 'store_or_None', SOURCE_STEP, 's', all_stops), + 'strict': ("Set strictness level", 'choice', 'store', DEFAULT_STRICT, strictness_options), }) self.log.debug("basic_options: descr %s opts %s" % (descr, opts)) @@ -187,7 +195,7 @@ def override_options(self): 'cleanup-builddir': ("Cleanup build dir after successful installation.", None, 'store_true', True), 'deprecated': ("Run pretending to be (future) version, to test removal of deprecated code.", None, 'store', None), - 'download-timeout': ("Timeout for initiating downloads (in seconds)", None, 'store', None), + 'download-timeout': ("Timeout for initiating downloads (in seconds)", float, 'store', None), 'easyblock': ("easyblock to use for processing the spec file or dumping the options", None, 'store', None, 'e', {'metavar': 'CLASS'}), 'experimental': ("Allow experimental code (with behaviour that can be changed/removed at any given time).", @@ -197,9 +205,11 @@ def override_options(self): 'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False), 'filter-deps': ("Comma separated list of dependencies that you DON'T want to install with EasyBuild, " "because equivalent OS packages are installed. (e.g. --filter-deps=zlib,ncurses)", - str, 'extend', None), - 'oldstyleconfig': ("Look for and use the oldstyle configuration file.", - None, 'store_true', True), + 'strlist', 'extend', None), + 'hide-deps': ("Comma separated list of dependencies that you want automatically hidden, " + "(e.g. --hide-deps=zlib,ncurses)", 'strlist', 'extend', None), + 'module-only': ("Only generate module file(s); skip all steps except for %s" % ', '.join(MODULE_ONLY_STEPS), + None, 'store_true', False), 'optarch': ("Set architecture optimization, overriding native architecture optimizations", None, 'store', None), 'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall"), @@ -228,10 +238,16 @@ def config_options(self): 'avail-repositories': ("Show all repository types (incl. non-usable)", None, "store_true", False,), 'buildpath': ("Temporary build path", None, 'store', mk_full_default_path('buildpath')), + 'external-modules-metadata': ("List of files specifying metadata for external modules (INI format)", + 'strlist', 'store', []), 'ignore-dirs': ("Directory names to ignore when searching for files/dirs", 'strlist', 'store', ['.git', '.svn']), 'installpath': ("Install path for software and modules", None, 'store', mk_full_default_path('installpath')), + 'installpath-modules': ("Install path for modules (if None, combine --installpath and --subdir-modules)", + None, 'store', None), + 'installpath-software': ("Install path for software (if None, combine --installpath and --subdir-software)", + None, 'store', None), 'job-backend': ("What job runner to use", 'choice', 'store', preferred_job_backend(), (avail_job_backends().keys())), # purposely take a copy for the default logfile format @@ -239,6 +255,8 @@ def config_options(self): 'strtuple', 'store', DEFAULT_LOGFILE_FORMAT[:], {'metavar': 'DIR,FORMAT'}), 'module-naming-scheme': ("Module naming scheme", 'choice', 'store', DEFAULT_MNS, sorted(avail_module_naming_schemes().keys())), + 'module-syntax': ("Syntax to be used for module files", 'choice', 'store', DEFAULT_MODULE_SYNTAX, + sorted(avail_module_generators().keys())), 'moduleclasses': (("Extend supported module classes " "(For more info on the default classes, use --show-default-moduleclasses)"), None, 'extend', [x[0] for x in DEFAULT_MODULECLASSES]), @@ -256,10 +274,7 @@ def config_options(self): 'repositorypath': (("Repository path, used by repository " "(is passed as list of arguments to create the repository instance). " "For more info, use --avail-repositories."), - 'strlist', 'store', - [mk_full_default_path('repositorypath')]), - 'show-default-moduleclasses': ("Show default module classes with description", - None, 'store_true', False), + 'strlist', 'store', self.default_repositorypath), 'sourcepath': ("Path(s) to where sources should be downloaded (string, colon-separated)", None, 'store', mk_full_default_path('sourcepath')), 'subdir-modules': ("Installpath subdir for modules", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_modules']), @@ -304,6 +319,9 @@ def informative_options(self): None, 'store', None, {'metavar': 'STR'}), 'search-short': ("Search for easyconfig files in the robot directory, print short paths", None, 'store', None, 'S', {'metavar': 'STR'}), + 'show-default-configfiles': ("Show list of default config files", None, 'store_true', False), + 'show-default-moduleclasses': ("Show default module classes with description", + None, 'store_true', False), }) self.log.debug("informative_options: descr %s opts %s" % (descr, opts)) @@ -361,27 +379,35 @@ def unittest_options(self): def validate(self): """Additional validation of options""" - error_cnt = 0 + error_msgs = [] for opt in ['software', 'try-software', 'toolchain', 'try-toolchain']: val = getattr(self.options, opt.replace('-', '_')) if val and len(val) != 2: - self.log.warning('--%s requires NAME,VERSION (given %s)' % (opt, ','.join(val))) - error_cnt += 1 + msg = "--%s requires NAME,VERSION (given %s)" % (opt, ','.join(val)) + error_msgs.append(msg) if self.options.umask: umask_regex = re.compile('^[0-7]{3}$') if not umask_regex.match(self.options.umask): - self.log.warning("--umask value should be 3 digits (0-7) (regex pattern '%s')" % umask_regex.pattern) - error_cnt += 1 + msg = "--umask value should be 3 digits (0-7) (regex pattern '%s')" % umask_regex.pattern + error_msgs.append(msg) + + # subdir options must be relative + for typ in ['modules', 'software']: + subdir_opt = 'subdir_%s' % typ + val = getattr(self.options, subdir_opt) + if os.path.isabs(getattr(self.options, subdir_opt)): + msg = "Configuration option '%s' must specify a *relative* path (use 'installpath-%s' instead?): '%s'" + msg = msg % (subdir_opt, typ, val) + error_msgs.append(msg) - if error_cnt > 0: - self.log.error("Found %s problems validating the options, treating warnings above as fatal." % error_cnt) + if error_msgs: + raise EasyBuildError("Found problems validating the options: %s", '\n'.join(error_msgs)) def postprocess(self): """Do some postprocessing, in particular print stuff""" build_log.EXPERIMENTAL = self.options.experimental - config.SUPPORT_OLDSTYLE = self.options.oldstyleconfig # set strictness of run module if self.options.strict: @@ -401,6 +427,7 @@ def postprocess(self): self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses, self.options.avail_repositories, self.options.show_default_moduleclasses, self.options.avail_modules_tools, self.options.avail_module_naming_schemes, + self.options.show_default_configfiles, ]): build_easyconfig_constants_dict() # runs the easyconfig constants sanity check self._postprocess_list_avail() @@ -408,20 +435,60 @@ def postprocess(self): # fail early if required dependencies for functionality requiring using GitHub API are not available: if self.options.from_pr or self.options.upload_test_report: if not HAVE_GITHUB_API: - self.log.error("Required support for using GitHub API is not available (see warnings).") + raise EasyBuildError("Required support for using GitHub API is not available (see warnings).") + + if self.options.module_syntax == ModuleGeneratorLua.SYNTAX and self.options.modules_tool != Lmod.__name__: + raise EasyBuildError("Generating Lua module files requires Lmod as modules tool.") # make sure a GitHub token is available when it's required if self.options.upload_test_report: if not HAVE_KEYRING: - self.log.error("Python 'keyring' module required for obtaining GitHub token is not available.") + raise EasyBuildError("Python 'keyring' module required for obtaining GitHub token is not available.") if self.options.github_user is None: - self.log.error("No GitHub user name provided, required for fetching GitHub token.") + raise EasyBuildError("No GitHub user name provided, required for fetching GitHub token.") token = fetch_github_token(self.options.github_user) if token is None: - self.log.error("Failed to obtain required GitHub token for user '%s'" % self.options.github_user) + raise EasyBuildError("Failed to obtain required GitHub token for user '%s'", self.options.github_user) + + self._postprocess_external_modules_metadata() self._postprocess_config() + def _postprocess_external_modules_metadata(self): + """Parse file(s) specifying metadata for external modules.""" + # leave external_modules_metadata untouched if no files are provided + if not self.options.external_modules_metadata: + self.log.debug("No metadata provided for external modules.") + return + + parsed_external_modules_metadata = ConfigObj() + for path in self.options.external_modules_metadata: + if os.path.exists(path): + self.log.debug("Parsing %s with external modules metadata", path) + try: + parsed_external_modules_metadata.merge(ConfigObj(path)) + except ConfigObjError, err: + raise EasyBuildError("Failed to parse %s with external modules metadata: %s", path, err) + else: + raise EasyBuildError("Specified path for file with external modules metadata does not exist: %s", path) + + # make sure name/version values are always lists, make sure they're equal length + for mod, entry in parsed_external_modules_metadata.items(): + for key in ['name', 'version']: + if isinstance(entry.get(key), basestring): + entry[key] = [entry[key]] + self.log.debug("Transformed external module metadata value %s for %s into a single-value list: %s", + key, mod, entry[key]) + + # if both names and versions are available, lists must be of same length + names, versions = entry.get('name'), entry.get('version') + if names is not None and versions is not None and len(names) != len(versions): + raise EasyBuildError("Different length for lists of names/versions in metadata for external module %s: " + "names: %s; versions: %s", mod, names, versions) + + self.options.external_modules_metadata = parsed_external_modules_metadata + self.log.debug("External modules metadata: %s", self.options.external_modules_metadata) + def _postprocess_config(self): """Postprocessing of configuration options""" if self.options.prefix is not None: @@ -432,7 +499,9 @@ def _postprocess_config(self): if dest == 'repository': setattr(self.options, dest, DEFAULT_REPOSITORY) elif dest == 'repositorypath': - setattr(self.options, dest, [mk_full_default_path(dest, prefix=self.options.prefix)]) + repositorypath = [mk_full_default_path(dest, prefix=self.options.prefix)] + setattr(self.options, dest, repositorypath) + self.go_cfg_constants[self.DEFAULTSECT]['DEFAULT_REPOSITORYPATH'] = repositorypath else: setattr(self.options, dest, mk_full_default_path(dest, prefix=self.options.prefix)) # LEGACY this line is here for oldstyle config reasons @@ -491,6 +560,10 @@ def _postprocess_list_avail(self): if self.options.avail_module_naming_schemes: msg += self.avail_list('module naming schemes', avail_module_naming_schemes()) + # dump default list of config files that are considered + if self.options.show_default_configfiles: + msg += self.show_default_configfiles() + # dump default moduleclasses with description if self.options.show_default_moduleclasses: msg += self.show_default_moduleclasses() @@ -600,7 +673,7 @@ def avail_toolchains(self): def avail_repositories(self): """Show list of known repository types.""" - repopath_defaults = mk_full_default_path('repositorypath') + repopath_defaults = self.default_repositorypath all_repos = avail_repositories(check_useable=False) usable_repos = avail_repositories(check_useable=True).keys() @@ -626,14 +699,34 @@ def avail_list(self, name, items): """Show list of available values passed by argument.""" return "List of supported %s:\n\t%s" % (name, '\n\t'.join(items)) + def show_default_configfiles(self): + """Show list of default config files.""" + xdg_config_home = os.environ.get('XDG_CONFIG_HOME', '(not set)') + xdg_config_dirs = os.environ.get('XDG_CONFIG_DIRS', '(not set)') + system_cfg_glob_paths = os.path.join('{' + ', '.join(XDG_CONFIG_DIRS) + '}', 'easybuild.d', '*.cfg') + found_cfgfile_cnt = len(self.DEFAULT_CONFIGFILES) + found_cfgfile_list = ', '.join(self.DEFAULT_CONFIGFILES) or '(none)' + lines = [ + "Default list of configuration files:", + '', + "[with $XDG_CONFIG_HOME: %s, $XDG_CONFIG_DIRS: %s]" % (xdg_config_home, xdg_config_dirs), + '', + "* user-level: %s" % os.path.join('${XDG_CONFIG_HOME:-$HOME/.config}', 'easybuild', 'config.cfg'), + " -> %s => %s" % (DEFAULT_USER_CFGFILE, ('not found', 'found')[os.path.exists(DEFAULT_USER_CFGFILE)]), + "* system-level: %s" % os.path.join('${XDG_CONFIG_DIRS:-/etc}', 'easybuild.d', '*.cfg'), + " -> %s => %s" % (system_cfg_glob_paths, ', '.join(DEFAULT_SYS_CFGFILES) or "(no matches)"), + '', + "Default list of existing configuration files (%d): %s" % (found_cfgfile_cnt, found_cfgfile_list), + ] + return '\n'.join(lines) + def show_default_moduleclasses(self): """Show list of default moduleclasses and description.""" - txt = ["Default available moduleclasses"] - indent = " " * 2 + lines = ["Default available module classes:", ''] maxlen = max([len(x[0]) for x in DEFAULT_MODULECLASSES]) + 1 # at least 1 space for name, descr in DEFAULT_MODULECLASSES: - txt.append("%s%s:%s%s" % (indent, name, (" " * (maxlen - len(name))), descr)) - return "\n".join(txt) + lines.append("\t%s:%s%s" % (name, (" " * (maxlen - len(name))), descr)) + return '\n'.join(lines) def parse_options(args=None): @@ -647,8 +740,12 @@ def parse_options(args=None): description = ("Builds software based on easyconfig (or parse a directory).\n" "Provide one or more easyconfigs or directories, use -H or --help more information.") - eb_go = EasyBuildOptions(usage=usage, description=description, prog='eb', envvar_prefix=CONFIG_ENV_VAR_PREFIX, - go_args=args) + try: + eb_go = EasyBuildOptions(usage=usage, description=description, prog='eb', envvar_prefix=CONFIG_ENV_VAR_PREFIX, + go_args=args, error_env_options=True, error_env_option_method=raise_easybuilderror) + except Exception, err: + raise EasyBuildError("Failed to parse configuration options: %s" % err) + return eb_go diff --git a/easybuild/tools/parallelbuild.py b/easybuild/tools/parallelbuild.py index 4397f8a6a6..082086dd2c 100644 --- a/easybuild/tools/parallelbuild.py +++ b/easybuild/tools/parallelbuild.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -203,4 +203,4 @@ def prepare_easyconfig(ec): easyblock_instance.close_log() os.remove(easyblock_instance.logfile) except (OSError, EasyBuildError), err: - _log.error("An error occured while preparing %s: %s" % (ec, err)) + raise EasyBuildError("An error occured while preparing %s: %s", ec, err) diff --git a/easybuild/tools/repository/__init__.py b/easybuild/tools/repository/__init__.py index 25f9cc1ca8..ebdc3c10b6 100644 --- a/easybuild/tools/repository/__init__.py +++ b/easybuild/tools/repository/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2014 Ghent University +# Copyright 2011-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/filerepo.py b/easybuild/tools/repository/filerepo.py index 4c1a58fe17..4cd314c08e 100644 --- a/easybuild/tools/repository/filerepo.py +++ b/easybuild/tools/repository/filerepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/gitrepo.py b/easybuild/tools/repository/gitrepo.py index ab6a5f07ca..83a1dae18c 100644 --- a/easybuild/tools/repository/gitrepo.py +++ b/easybuild/tools/repository/gitrepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -43,8 +43,10 @@ import time from vsc.utils import fancylogger +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import rmtree2 from easybuild.tools.repository.filerepo import FileRepository +from easybuild.tools.version import VERSION _log = fancylogger.getLogger('gitrepo', fname=False) @@ -87,7 +89,7 @@ def setup_repo(self): try: git.GitCommandError except NameError, err: - self.log.exception("It seems like GitPython is not available: %s" % err) + raise EasyBuildError("It seems like GitPython is not available: %s", err) self.wc = tempfile.mkdtemp(prefix='git-wc-') @@ -105,7 +107,7 @@ def create_working_copy(self): self.log.debug("rep name is %s" % reponame) except git.GitCommandError, err: # it might already have existed - self.log.warning("Git local repo initialization failed, it might already exist: %s" % err) + self.log.warning("Git local repo initialization failed, it might already exist: %s", err) # local repo should now exist, let's connect to it again try: @@ -113,14 +115,14 @@ def create_working_copy(self): self.log.debug("connectiong to git repo in %s" % self.wc) self.client = git.Git(self.wc) except (git.GitCommandError, OSError), err: - self.log.error("Could not create a local git repo in wc %s: %s" % (self.wc, err)) + raise EasyBuildError("Could not create a local git repo in wc %s: %s", self.wc, err) # try to get the remote data in the local repo try: res = self.client.pull() self.log.debug("pulled succesfully to %s in %s" % (res, self.wc)) except (git.GitCommandError, OSError), err: - self.log.exception("pull in working copy %s went wrong: %s" % (self.wc, err)) + raise EasyBuildError("pull in working copy %s went wrong: %s", self.wc, err) def add_easyconfig(self, cfg, name, version, stats, append): """ @@ -138,25 +140,24 @@ def commit(self, msg=None): """ Commit working copy to git repository """ - self.log.debug("committing in git: %s" % msg) - completemsg = "EasyBuild-commit from %s (time: %s, user: %s) \n%s" % (socket.gethostname(), - time.strftime("%Y-%m-%d_%H-%M-%S"), - getpass.getuser(), - msg) + host = socket.gethostname() + timestamp = time.strftime("%Y-%m-%d_%H-%M-%S") + user = getpass.getuser() + completemsg = "%s with EasyBuild v%s @ %s (time: %s, user: %s)" % (msg, VERSION, host, timestamp, user) + self.log.debug("committing in git with message: %s" % msg) + self.log.debug("git status: %s" % self.client.status()) try: - self.client.commit('-am "%s"' % completemsg) - self.log.debug("succesfull commit") + self.client.commit('-am %s' % completemsg) + self.log.debug("succesfull commit: %s", self.client.log('HEAD^!')) except GitCommandError, err: - self.log.warning("Commit from working copy %s (msg: %s) failed, empty commit?\n%s" % (self.wc, msg, err)) + self.log.warning("Commit from working copy %s failed, empty commit? (msg: %s): %s", self.wc, msg, err) try: info = self.client.push() - self.log.debug("push info: %s " % info) + self.log.debug("push info: %s ", info) except GitCommandError, err: - self.log.warning("Push from working copy %s to remote %s (msg: %s) failed: %s" % (self.wc, - self.repo, - msg, - err)) + self.log.warning("Push from working copy %s to remote %s failed (msg: %s): %s", + self.wc, self.repo, msg, err) def cleanup(self): """ @@ -166,4 +167,4 @@ def cleanup(self): self.wc = os.path.dirname(self.wc) rmtree2(self.wc) except IOError, err: - self.log.exception("Can't remove working copy %s: %s" % (self.wc, err)) + raise EasyBuildError("Can't remove working copy %s: %s", self.wc, err) diff --git a/easybuild/tools/repository/repository.py b/easybuild/tools/repository/repository.py index a8dca02326..afb44f7e20 100644 --- a/easybuild/tools/repository/repository.py +++ b/easybuild/tools/repository/repository.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,6 +37,7 @@ from vsc.utils import fancylogger from vsc.utils.missing import get_subclasses +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.utilities import import_available_modules _log = fancylogger.getLogger('repository', fname=False) @@ -126,7 +127,7 @@ def avail_repositories(check_useable=True): class_dict = dict([(x.__name__, x) for x in get_subclasses(Repository) if x.USABLE or not check_useable]) if not 'FileRepository' in class_dict: - _log.error('avail_repositories: FileRepository missing from list of repositories') + raise EasyBuildError("avail_repositories: FileRepository missing from list of repositories") return class_dict @@ -144,13 +145,13 @@ def init_repository(repository, repository_path): elif isinstance(repository_path, (tuple, list)) and len(repository_path) <= 2: inited_repo = repo(*repository_path) else: - _log.error('repository_path should be a string or list/tuple of maximum 2 elements (current: %s, type %s)' % - (repository_path, type(repository_path))) + raise EasyBuildError("repository_path should be a string or list/tuple of maximum 2 elements " + "(current: %s, type %s)", repository_path, type(repository_path)) except Exception, err: - _log.error('Failed to create a repository instance for %s (class %s) with args %s (msg: %s)' % - (repository, repo.__name__, repository_path, err)) + raise EasyBuildError("Failed to create a repository instance for %s (class %s) with args %s (msg: %s)", + repository, repo.__name__, repository_path, err) else: - _log.error('Unknown typo of repository spec: %s (type %s)' % (repo, type(repo))) + raise EasyBuildError("Unknown typo of repository spec: %s (type %s)", repo, type(repo)) inited_repo.init() return inited_repo diff --git a/easybuild/tools/repository/svnrepo.py b/easybuild/tools/repository/svnrepo.py index 31f38abdc9..7afc3dd74b 100644 --- a/easybuild/tools/repository/svnrepo.py +++ b/easybuild/tools/repository/svnrepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -43,6 +43,7 @@ import time from vsc.utils import fancylogger +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import rmtree2 from easybuild.tools.repository.filerepo import FileRepository @@ -88,8 +89,8 @@ def setup_repo(self): try: pysvn.ClientError # IGNORE:E0611 pysvn fails to recognize ClientError is available except NameError, err: - self.log.exception("pysvn not available (%s). Make sure it is installed " % err + - "properly. Run 'python -c \"import pysvn\"' to test.") + raise EasyBuildError("pysvn not available (%s). Make sure it is installed properly. " + "Run 'python -c \"import pysvn\"' to test.", err) # try to connect to the repository self.log.debug("Try to connect to repository %s" % self.repo) @@ -97,13 +98,13 @@ def setup_repo(self): self.client = pysvn.Client() self.client.exception_style = 0 except ClientError: - self.log.exception("Svn Client initialization failed.") + raise EasyBuildError("Svn Client initialization failed.") try: if not self.client.is_url(self.repo): - self.log.error("Provided repository %s is not a valid svn url" % self.repo) + raise EasyBuildError("Provided repository %s is not a valid svn url", self.repo) except ClientError: - self.log.exception("Can't connect to svn repository %s" % self.repo) + raise EasyBuildError("Can't connect to svn repository %s", self.repo) def create_working_copy(self): """ @@ -116,16 +117,16 @@ def create_working_copy(self): try: self.client.info2(self.repo, recurse=False) except ClientError: - self.log.exception("Getting info from %s failed." % self.wc) + raise EasyBuildError("Getting info from %s failed.", self.wc) try: res = self.client.update(self.wc) self.log.debug("Updated to revision %s in %s" % (res, self.wc)) except ClientError: - self.log.exception("Update in wc %s went wrong" % self.wc) + raise EasyBuildError("Update in wc %s went wrong", self.wc) if len(res) == 0: - self.log.error("Update returned empy list (working copy: %s)" % (self.wc)) + raise EasyBuildError("Update returned empy list (working copy: %s)", self.wc) if res[0].number == -1: # revision number of update is -1 @@ -134,7 +135,7 @@ def create_working_copy(self): res = self.client.checkout(self.repo, self.wc) self.log.debug("Checked out revision %s in %s" % (res.number, self.wc)) except ClientError, err: - self.log.exception("Checkout of path / in working copy %s went wrong: %s" % (self.wc, err)) + raise EasyBuildError("Checkout of path / in working copy %s went wrong: %s", self.wc, err) def add_easyconfig(self, cfg, name, version, stats, append): """ @@ -160,7 +161,7 @@ def commit(self, msg=None): try: self.client.checkin(self.wc, completemsg, recurse=True) except ClientError, err: - self.log.exception("Commit from working copy %s (msg: %s) failed: %s" % (self.wc, msg, err)) + raise EasyBuildError("Commit from working copy %s (msg: %s) failed: %s", self.wc, msg, err) def cleanup(self): """ @@ -169,4 +170,4 @@ def cleanup(self): try: rmtree2(self.wc) except OSError, err: - self.log.exception("Can't remove working copy %s: %s" % (self.wc, err)) + raise EasyBuildError("Can't remove working copy %s: %s", self.wc, err) diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index 99a3f6b9fa..9b82878931 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -38,6 +38,7 @@ from easybuild.framework.easyconfig.easyconfig import ActiveMNS, process_easyconfig, robot_find_easyconfig from easybuild.framework.easyconfig.tools import find_resolved_modules, skip_available +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option from easybuild.tools.filetools import det_common_path_prefix, search_file from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS @@ -152,9 +153,8 @@ def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False): # make sure this stops, we really don't want to get stuck in an infinite loop loopcnt += 1 if loopcnt > maxloopcnt: - tup = (maxloopcnt, unprocessed, irresolvable) - msg = "Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)" % tup - _log.error(msg) + raise EasyBuildError("Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)", + maxloopcnt, unprocessed, irresolvable) # first try resolving dependencies without using external dependencies last_processed_count = -1 @@ -166,6 +166,13 @@ def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False): if not ec['full_mod_name'] in [x['full_mod_name'] for x in ordered_ecs]: ordered_ecs.append(ec) + # dependencies marked as external modules should be resolved via available modules at this point + missing_external_modules = [d['full_mod_name'] for ec in unprocessed for d in ec['dependencies'] + if d.get('external_module', False)] + if missing_external_modules: + raise EasyBuildError("Missing modules for one or more dependencies marked as external modules: %s", + missing_external_modules) + # robot: look for existing dependencies, add them if robot and unprocessed: @@ -205,8 +212,8 @@ def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False): mods = [spec['ec'].full_mod_name for spec in processed_ecs] dep_mod_name = ActiveMNS().det_full_module_name(cand_dep) if not dep_mod_name in mods: - tup = (path, dep_mod_name, mods) - _log.error("easyconfig file %s does not contain module %s (mods: %s)" % tup) + raise EasyBuildError("easyconfig file %s does not contain module %s (mods: %s)", + path, dep_mod_name, mods) for ec in processed_ecs: if not ec in unprocessed + additional: @@ -218,6 +225,7 @@ def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False): # add additional (new) easyconfigs to list of stuff to process unprocessed.extend(additional) + _log.debug("Unprocessed dependencies: %s", unprocessed) elif not robot: # no use in continuing if robot is not enabled, dependencies won't be resolved anyway @@ -229,9 +237,9 @@ def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False): irresolvable_mods_eb = [EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable] _log.warning("Irresolvable dependencies (EasyBuild module names): %s" % ', '.join(irresolvable_mods_eb)) irresolvable_mods = [ActiveMNS().det_full_module_name(dep) for dep in irresolvable] - _log.error('Irresolvable dependencies encountered: %s' % ', '.join(irresolvable_mods)) + raise EasyBuildError("Irresolvable dependencies encountered: %s", ', '.join(irresolvable_mods)) - _log.info("Dependency resolution complete, building as follows:\n%s" % ordered_ecs) + _log.info("Dependency resolution complete, building as follows: %s" % ordered_ecs) return ordered_ecs diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index ba13928464..651042651e 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -43,7 +43,7 @@ from vsc.utils import fancylogger from easybuild.tools.asyncprocess import PIPE, STDOUT, Popen, recv_some, send_all -import easybuild.tools.build_log # this import is required to obtain a correct (EasyBuild) logger! +from easybuild.tools.build_log import EasyBuildError _log = fancylogger.getLogger('run', fname=False) @@ -60,35 +60,12 @@ strictness = WARN -def adjust_cmd(func): - """Make adjustments to given command, if required.""" - - def inner(cmd, *args, **kwargs): - # SuSE hack - # - profile is not resourced, and functions (e.g. module) is not inherited - if 'PROFILEREAD' in os.environ and (len(os.environ['PROFILEREAD']) > 0): - filepaths = ['/etc/profile.d/modules.sh'] - extra = '' - for fp in filepaths: - if os.path.exists(fp): - extra = ". %s &&%s" % (fp, extra) - else: - _log.warning("Can't find file %s" % fp) - - cmd = "%s %s" % (extra, cmd) - - return func(cmd, *args, **kwargs) - - return inner - - -@adjust_cmd def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None): """ Executes a command cmd - returns exitcode and stdout+stderr (mixed) - no input though stdin - - if log_ok or log_all are set -> will log.error if non-zero exit-code + - if log_ok or log_all are set -> will raise EasyBuildError if non-zero exit-code - if simple is True -> instead of returning a tuple (output, ec) it will just return True or False signifying succes - inp is the input given to the command - regexp -> Regex used to check the output for errors. If True will use default (see parselogForError) @@ -119,7 +96,7 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, close_fds=True, executable="/bin/bash") except OSError, err: - _log.error("run_cmd init cmd %s failed:%s" % (cmd, err)) + raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err) if inp: p.stdin.write(inp) p.stdin.close() @@ -148,12 +125,11 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True try: os.chdir(cwd) except OSError, err: - _log.error("Failed to return to %s after executing command: %s" % (cwd, err)) + raise EasyBuildError("Failed to return to %s after executing command: %s", cwd, err) return parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp) -@adjust_cmd def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None): """ Executes a command cmd @@ -200,15 +176,15 @@ def process_QA(q, a_s): if regQ.search(q): return (a_s, regQ) else: - _log.error("runqanda: Question %s converted in %s does not match itself" % (q, regQtxt)) + raise EasyBuildError("runqanda: Question %s converted in %s does not match itself", q, regQtxt) def check_answers_list(answers): """Make sure we have a list of answers (as strings).""" if isinstance(answers, basestring): answers = [answers] elif not isinstance(answers, list): - msg = "Invalid type for answer on %s, no string or list: %s (%s)" % (question, type(answers), answers) - _log.error(msg) + raise EasyBuildError("Invalid type for answer on %s, no string or list: %s (%s)", + question, type(answers), answers) # list is manipulated when answering matching question, so return a copy return answers[:] @@ -247,7 +223,7 @@ def check_answers_list(answers): _log.debug('run_cmd_qa: Command output will be logged to %s' % runLog.name) runLog.write(cmd + "\n\n") except IOError, err: - _log.error("Opening log file for Q&A failed: %s" % err) + raise EasyBuildError("Opening log file for Q&A failed: %s", err) else: runLog = None @@ -256,7 +232,7 @@ def check_answers_list(answers): try: p = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT, stdin=PIPE, close_fds=True, executable="/bin/bash") except OSError, err: - _log.error("run_cmd_qa init cmd %s failed:%s" % (cmd, err)) + raise EasyBuildError("run_cmd_qa init cmd %s failed:%s", cmd, err) ec = p.poll() stdoutErr = '' @@ -328,8 +304,8 @@ def check_answers_list(answers): except OSError, err: _log.debug("run_cmd_qa exception caught when killing child process: %s" % err) _log.debug("run_cmd_qa: full stdouterr: %s" % stdoutErr) - _log.error("run_cmd_qa: cmd %s : Max nohits %s reached: end of output %s" % - (cmd, maxHitCount, stdoutErr[-500:])) + raise EasyBuildError("run_cmd_qa: cmd %s : Max nohits %s reached: end of output %s", + cmd, maxHitCount, stdoutErr[-500:]) # the sleep below is required to avoid exiting on unknown 'questions' too early (see above) time.sleep(1) @@ -351,7 +327,7 @@ def check_answers_list(answers): try: os.chdir(cwd) except OSError, err: - _log.error("Failed to return to %s after executing command: %s" % (cwd, err)) + raise EasyBuildError("Failed to return to %s after executing command: %s", cwd, err) return parse_cmd_output(cmd, stdoutErr, ec, simple, log_all, log_ok, regexp) @@ -370,7 +346,7 @@ def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp): check_ec = True use_regexp = True else: - _log.error("invalid strictness setting: %s" % strictness) + raise EasyBuildError("invalid strictness setting: %s", strictness) # allow for overriding the regexp setting if not regexp: @@ -381,7 +357,7 @@ def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp): if ec and (log_all or log_ok): # We don't want to error if the user doesn't care if check_ec: - _log.error('cmd "%s" exited with exitcode %s and output:\n%s' % (cmd, ec, stdouterr)) + raise EasyBuildError('cmd "%s" exited with exitcode %s and output:\n%s', cmd, ec, stdouterr) else: _log.warn('cmd "%s" exited with exitcode %s and output:\n%s' % (cmd, ec, stdouterr)) elif not ec: @@ -394,7 +370,7 @@ def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp): if len(res) > 0: message = "Found %s errors in command output (output: %s)" % (len(res), ", ".join([r[0] for r in res])) if use_regexp: - _log.error(message) + raise EasyBuildError(message) else: _log.warn(message) @@ -424,7 +400,7 @@ def parse_log_for_error(txt, regExp=None, stdout=True, msg=None): elif type(regExp) == str: pass else: - _log.error("parse_log_for_error no valid regExp used: %s" % regExp) + raise EasyBuildError("parse_log_for_error no valid regExp used: %s", regExp) reg = re.compile(regExp, re.I) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 921c8e926e..e3bd6316e9 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2014 Ghent University +# Copyright 2011-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -38,6 +38,7 @@ from vsc.utils import fancylogger from vsc.utils.affinity import sched_getaffinity +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file, which from easybuild.tools.run import run_cmd @@ -121,8 +122,8 @@ def get_cpu_vendor(): arch = res.group('vendor') if arch in VENDORS: vendor = VENDORS[arch] - tup = (vendor, vendor_regex.pattern, PROC_CPUINFO_FP) - _log.debug("Determined CPU vendor on Linux as being '%s' via regex '%s' in %s" % tup) + _log.debug("Determined CPU vendor on Linux as being '%s' via regex '%s' in %s", + vendor, vendor_regex.pattern, PROC_CPUINFO_FP) elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.vendor" @@ -157,8 +158,8 @@ def get_cpu_family(): power_regex = re.compile(r"^cpu\s+:\s*POWER.*", re.M) if power_regex.search(cpuinfo_txt): family = POWER - tup = (power_regex.pattern, PROC_CPUINFO_FP, family) - _log.debug("Determined CPU family using regex '%s' in %s: %s" % tup) + _log.debug("Determined CPU family using regex '%s' in %s: %s", + power_regex.pattern, PROC_CPUINFO_FP, family) if family is None: family = UNKNOWN @@ -182,8 +183,8 @@ def get_cpu_model(): res = model_regex.search(txt) if res is not None: model = res.group('model').strip() - tup = (model_regex.pattern, PROC_CPUINFO_FP, model) - _log.debug("Determined CPU model on Linux using regex '%s' in %s: %s" % tup) + _log.debug("Determined CPU model on Linux using regex '%s' in %s: %s", + model_regex.pattern, PROC_CPUINFO_FP, model) elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.brand_string" @@ -348,7 +349,7 @@ def get_os_version(): if not known_sp: suff = '_UNKNOWN_SP' else: - _log.error("Don't know how to determine subversions for SLES %s" % os_version) + raise EasyBuildError("Don't know how to determine subversions for SLES %s", os_version) return os_version else: @@ -413,8 +414,8 @@ def get_glibc_version(): _log.debug("Found glibc version %s" % glibc_version) return glibc_version else: - tup = (glibc_ver_str, glibc_ver_regex.pattern) - _log.error("Failed to determine glibc version from '%s' using pattern '%s'." % tup) + raise EasyBuildError("Failed to determine glibc version from '%s' using pattern '%s'.", + glibc_ver_str, glibc_ver_regex.pattern) else: # no glibc on OS X standard _log.debug("No glibc on a non-Linux system, so can't determine version.") @@ -447,7 +448,7 @@ def use_group(group_name): try: group_id = grp.getgrnam(group_name).gr_gid except KeyError, err: - _log.error("Failed to get group ID for '%s', group does not exist (err: %s)" % (group_name, err)) + raise EasyBuildError("Failed to get group ID for '%s', group does not exist (err: %s)", group_name, err) group = (group_name, group_id) try: @@ -460,7 +461,7 @@ def use_group(group_name): err_msg += "change the primary group before using EasyBuild, using 'newgrp %s'." % group_name else: err_msg += "current user '%s' is not in group %s (members: %s)" % (user, group, grp_members) - _log.error(err_msg) + raise EasyBuildError(err_msg) _log.info("Using group '%s' (gid: %s)" % group) return group @@ -476,7 +477,7 @@ def det_parallelism(par, maxpar): try: par = int(par) except ValueError, err: - _log.error("Specified level of parallelism '%s' is not an integer value: %s" % (par, err)) + raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err) else: par = get_avail_core_count() # check ulimit -u @@ -491,7 +492,7 @@ def det_parallelism(par, maxpar): par = par_guess _log.info("Limit parallel builds to %s because max user processes is %s" % (par, out)) except ValueError, err: - _log.exception("Failed to determine max user processes (%s, %s): %s" % (ec, out, err)) + raise EasyBuildError("Failed to determine max user processes (%s, %s): %s", ec, out, err) if maxpar is not None and maxpar < par: _log.info("Limiting parallellism from %s to %s" % (par, maxpar)) diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index 683f607e4a..e42327e831 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -96,7 +96,7 @@ def regtest(easyconfig_paths, build_specs=None): for path in easyconfig_paths: ecfiles += find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs')) else: - _log.error("No easyconfig paths specified.") + raise EasyBuildError("No easyconfig paths specified.") test_results = [] diff --git a/easybuild/tools/toolchain/__init__.py b/easybuild/tools/toolchain/__init__.py index fb2a99459a..4c2abb3e18 100644 --- a/easybuild/tools/toolchain/__init__.py +++ b/easybuild/tools/toolchain/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index fda05be513..45de8f34f7 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,10 +28,8 @@ @author: Stijn De Weirdt (Ghent University) @author: Kenneth Hoste (Ghent University) """ - -import os - from easybuild.tools import systemtools +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option from easybuild.tools.toolchain.constants import COMPILER_VARIABLES from easybuild.tools.toolchain.toolchain import Toolchain @@ -157,10 +155,8 @@ def _set_compiler_toolchainoptions(self): getattr(self, 'COMPILER_%sUNIQUE_OPTS' % infix, None), getattr(self, 'COMPILER_%sUNIQUE_OPTION_MAP' % infix, None), ) - #print "added options for prefix %s" % prefix - # redefine optarch - self._get_optimal_architecture() + self._set_optimal_architecture() def _set_compiler_vars(self): """Set the compiler variables""" @@ -186,7 +182,7 @@ def _set_compiler_vars(self): # only warn if prefix is set, not all languages may be supported (e.g., no Fortran for CUDA) self.log.warn("_set_compiler_vars: %s compiler variable %s undefined" % (prefix, var)) else: - self.log.raiseException("_set_compiler_vars: compiler variable %s undefined" % var) + raise EasyBuildError("_set_compiler_vars: compiler variable %s undefined", var) self.variables[pref_var] = value if is32bit: @@ -257,7 +253,7 @@ def _set_compiler_flags(self): self.variables.nappend('F90FLAGS', fflags) self.variables.join('F90FLAGS', 'OPTFLAGS', 'PRECFLAGS') - def _get_optimal_architecture(self): + def _set_optimal_architecture(self): """ Get options for the current architecture """ if self.arch is None: self.arch = systemtools.get_cpu_family() @@ -269,11 +265,11 @@ def _get_optimal_architecture(self): optarch = self.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[self.arch] if optarch is not None: - self.log.info("_get_optimal_architecture: using %s as optarch for %s." % (optarch, self.arch)) + self.log.info("_set_optimal_architecture: using %s as optarch for %s." % (optarch, self.arch)) self.options.options_map['optarch'] = optarch if 'optarch' in self.options.options_map and self.options.options_map.get('optarch', None) is None: - self.log.raiseException("_get_optimal_architecture: don't know how to set optarch for %s." % self.arch) + raise EasyBuildError("_set_optimal_architecture: don't know how to set optarch for %s", self.arch) def comp_family(self, prefix=None): """ @@ -286,4 +282,4 @@ def comp_family(self, prefix=None): if comp_family: return comp_family else: - self.log.raiseException('comp_family: COMPILER_%sFAMILY is undefined.' % infix) + raise EasyBuildError("comp_family: COMPILER_%sFAMILY is undefined", infix) diff --git a/easybuild/tools/toolchain/constants.py b/easybuild/tools/toolchain/constants.py index ddf0f9f4c5..7c9cbf30d1 100644 --- a/easybuild/tools/toolchain/constants.py +++ b/easybuild/tools/toolchain/constants.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/fft.py b/easybuild/tools/toolchain/fft.py index 192e418d06..f665e35f40 100644 --- a/easybuild/tools/toolchain/fft.py +++ b/easybuild/tools/toolchain/fft.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/linalg.py b/easybuild/tools/toolchain/linalg.py index 7086b0de4c..15b091ebba 100644 --- a/easybuild/tools/toolchain/linalg.py +++ b/easybuild/tools/toolchain/linalg.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -29,6 +29,7 @@ @author: Kenneth Hoste (Ghent University) """ +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.toolchain import Toolchain @@ -95,7 +96,7 @@ def set_variables(self): def _set_blas_variables(self): """Set BLAS related variables""" if self.BLAS_LIB is None: - self.log.raiseException("_set_blas_variables: BLAS_LIB not set") + raise EasyBuildError("_set_blas_variables: BLAS_LIB not set") self.BLAS_LIB = self.variables.nappend('LIBBLAS', [x % self.BLAS_LIB_MAP for x in self.BLAS_LIB]) self.variables.add_begin_end_linkerflags(self.BLAS_LIB, @@ -143,7 +144,7 @@ def _set_lapack_variables(self): self.variables.join('LAPACK_INC_DIR', 'BLAS_INC_DIR') else: if self.LAPACK_LIB is None: - self.log.raiseException("_set_lapack_variables: LAPACK_LIB not set") + raise EasyBuildError("_set_lapack_variables: LAPACK_LIB not set") self.LAPACK_LIB = self.variables.nappend('LIBLAPACK_ONLY', self.LAPACK_LIB) self.variables.add_begin_end_linkerflags(self.LAPACK_LIB, toggle_startstopgroup=self.LAPACK_LIB_GROUP, @@ -222,7 +223,7 @@ def _set_scalapack_variables(self): """Set ScaLAPACK related variables""" if self.SCALAPACK_LIB is None: - self.log.raiseException("_set_blas_variables: SCALAPACK_LIB not set") + raise EasyBuildError("_set_blas_variables: SCALAPACK_LIB not set") lib_map = {} if hasattr(self, 'BLAS_LIB_MAP') and self.BLAS_LIB_MAP is not None: @@ -268,7 +269,7 @@ def _set_scalapack_variables(self): if getattr(self, 'LIB_MULTITHREAD', None) is not None: self.variables.nappend('LIBSCALAPACK_MT', self.LIB_MULTITHREAD) else: - self.log.raiseException("_set_scalapack_variables: LIBSCALAPACK without SCALAPACK_REQUIRES not implemented") + raise EasyBuildError("_set_scalapack_variables: LIBSCALAPACK without SCALAPACK_REQUIRES not implemented") self.variables.join('SCALAPACK_STATIC_LIBS', 'LIBSCALAPACK') @@ -279,4 +280,3 @@ def _set_scalapack_variables(self): self._add_dependency_variables(self.SCALAPACK_MODULE_NAME, ld=self.SCALAPACK_LIB_DIR, cpp=self.SCALAPACK_INCLUDE_DIR) - diff --git a/easybuild/tools/toolchain/mpi.py b/easybuild/tools/toolchain/mpi.py index 7a9694de8c..74da9d5ff6 100644 --- a/easybuild/tools/toolchain/mpi.py +++ b/easybuild/tools/toolchain/mpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -33,6 +33,7 @@ import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import write_file from easybuild.tools.toolchain.constants import COMPILER_VARIABLES, MPI_COMPILER_TEMPLATE, SEQ_COMPILER_TEMPLATE from easybuild.tools.toolchain.toolchain import Toolchain @@ -106,7 +107,7 @@ def _set_mpi_compiler_variables(self): value = getattr(self, 'MPI_COMPILER_%s' % var.upper(), None) if value is None: - self.log.raiseException("_set_mpi_compiler_variables: mpi compiler variable %s undefined" % var) + raise EasyBuildError("_set_mpi_compiler_variables: mpi compiler variable %s undefined", var) self.variables.nappend_el(var, value) # complete compiler variable template to produce e.g. 'mpicc -cc=icc -X -Y' from 'mpicc -cc=%(CC_base)' @@ -159,7 +160,7 @@ def mpi_family(self): if self.MPI_FAMILY: return self.MPI_FAMILY else: - self.log.raiseException("mpi_family: MPI_FAMILY is undefined.") + raise EasyBuildError("mpi_family: MPI_FAMILY is undefined.") # FIXME: deprecate this function, use mympirun instead # this requires that either mympirun is packaged together with EasyBuild, or that vsc-tools is a dependency of EasyBuild @@ -188,10 +189,16 @@ def mpi_cmd_for(self, cmd, nr_ranks): # Intel MPI mpirun needs more work if mpi_family == toolchain.INTELMPI: # @UndefinedVariable - tmpdir = tempfile.mkdtemp(prefix='eb-mpi_cmd_for-') - # set temporary dir for mdp - env.setvar('I_MPI_MPD_TMPDIR', tmpdir) + # note: this needs to be kept *short*, to avoid mpirun failing with "socket.error: AF_UNIX path too long" + # exact limit is unknown, but ~20 characters seems to be OK + env.setvar('I_MPI_MPD_TMPDIR', tempfile.gettempdir()) + mpd_tmpdir = os.environ['I_MPI_MPD_TMPDIR'] + if len(mpd_tmpdir) > 20: + self.log.warning("$I_MPI_MPD_TMPDIR should be (very) short to avoid problems: %s" % mpd_tmpdir) + + # temporary location for mpdboot and nodes files + tmpdir = tempfile.mkdtemp(prefix='mpi_cmd_for-') # set PBS_ENVIRONMENT, so that --file option for mpdboot isn't stripped away env.setvar('PBS_ENVIRONMENT', "PBS_BATCH_MPI") @@ -207,7 +214,7 @@ def mpi_cmd_for(self, cmd, nr_ranks): os.remove(fn) write_file(fn, "localhost ifhn=localhost") except OSError, err: - self.log.error("Failed to create file %s: %s" % (fn, err)) + raise EasyBuildError("Failed to create file %s: %s", fn, err) params.update({'mpdbf': "--file=%s" % fn}) @@ -218,11 +225,11 @@ def mpi_cmd_for(self, cmd, nr_ranks): os.remove(fn) write_file(fn, "localhost\n" * nr_ranks) except OSError, err: - self.log.error("Failed to create file %s: %s" % (fn, err)) + raise EasyBuildError("Failed to create file %s: %s", fn, err) params.update({'nodesfile': "-machinefile %s" % fn}) if mpi_family in mpi_cmds.keys(): return mpi_cmds[mpi_family] % params else: - self.log.error("Don't know how to create an MPI command for MPI library of type '%s'." % mpi_family) + raise EasyBuildError("Don't know how to create an MPI command for MPI library of type '%s'.", mpi_family) diff --git a/easybuild/tools/toolchain/options.py b/easybuild/tools/toolchain/options.py index 9b0a5faaa8..32b162151f 100644 --- a/easybuild/tools/toolchain/options.py +++ b/easybuild/tools/toolchain/options.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,6 +37,8 @@ from vsc.utils import fancylogger +from easybuild.tools.build_log import EasyBuildError + class ToolchainOptions(dict): def __init__(self): @@ -62,9 +64,9 @@ def _add_options(self, options): self.log.debug("_add_options: adding options %s" % options) for name, value in options.items(): if not isinstance(value, (list, tuple,)) and len(value) == 2: - self.log.raiseException("_add_options: option name %s has to be 2 element list (%s)" % (name, value)) + raise EasyBuildError("_add_options: option name %s has to be 2 element list (%s)", name, value) if name in self: - self.log.debug("_add_options: redefining previous name %s (previous value %s)" % (name, self.get(name))) + self.log.debug("_add_options: redefining previous name %s (previous value %s)", name, self.get(name)) self.__setitem__(name, value[0]) self.description.__setitem__(name, value[1]) @@ -75,9 +77,9 @@ def _add_options_map(self, options_map): for name in options_map.keys(): if not name in self: if name.startswith('_opt_'): - self.log.debug("_add_options_map: no option with name %s defined, but allowed" % name) + self.log.debug("_add_options_map: no option with name %s defined, but allowed", name) else: - self.log.raiseException("_add_options_map: no option with name %s defined" % name) + raise EasyBuildError("_add_options_map: no option with name %s defined", name) self.options_map.update(options_map) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 6b9924858e..a5cad572be 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -32,12 +32,14 @@ """ import os -import re from vsc.utils import fancylogger +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, install_path from easybuild.tools.environment import setvar -from easybuild.tools.modules import get_software_root, get_software_version, modules_tool +from easybuild.tools.module_generator import dependencies_for +from easybuild.tools.modules import get_software_root, get_software_root_env_var_name +from easybuild.tools.modules import get_software_version, get_software_version_env_var_name, modules_tool from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME, DUMMY_TOOLCHAIN_VERSION from easybuild.tools.toolchain.options import ToolchainOptions from easybuild.tools.toolchain.toolchainvariables import ToolchainVariables @@ -54,6 +56,7 @@ class Toolchain(object): NAME = None VERSION = None + TOOLCHAIN_FAMILY = None # class method def _is_toolchain_for(cls, name): @@ -81,13 +84,13 @@ def __init__(self, name=None, version=None, mns=None): if name is None: name = self.NAME if name is None: - self.log.error("Toolchain init: no name provided") + raise EasyBuildError("Toolchain init: no name provided") self.name = name if version is None: version = self.VERSION if version is None: - self.log.error("Toolchain init: no version provided") + raise EasyBuildError("Toolchain init: no version provided") self.version = version self.vars = None @@ -128,7 +131,7 @@ def get_variable(self, name, typ=str): elif typ == list: return self.variables[name].flatten() else: - self.log.error("get_variable: Don't know how to create value of type %s." % typ) + raise EasyBuildError("get_variable: Don't know how to create value of type %s.", typ) def set_variables(self): """Do nothing? Everything should have been set by others @@ -187,18 +190,18 @@ def _get_software_root(self, name): """Try to get the software root for name""" root = get_software_root(name) if root is None: - self.log.error("get_software_root software root for %s was not found in environment" % name) + raise EasyBuildError("get_software_root software root for %s was not found in environment", name) else: - self.log.debug("get_software_root software root %s for %s was found in environment" % (root, name)) + self.log.debug("get_software_root software root %s for %s was found in environment", root, name) return root def _get_software_version(self, name): """Try to get the software root for name""" version = get_software_version(name) if version is None: - self.log.error("get_software_version software version for %s was not found in environment" % name) + raise EasyBuildError("get_software_version software version for %s was not found in environment", name) else: - self.log.debug("get_software_version software version %s for %s was found in environment" % (version, name)) + self.log.debug("get_software_version software version %s for %s was found in environment", version, name) return version @@ -221,7 +224,7 @@ def as_dict(self, name=None, version=None): def det_short_module_name(self): """Determine module name for this toolchain.""" if self.mod_short_name is None: - self.log.error("Toolchain module name was not set yet (using set_module_info).") + raise EasyBuildError("Toolchain module name was not set yet (using set_module_info).") return self.mod_short_name def _toolchain_exists(self): @@ -234,7 +237,7 @@ def _toolchain_exists(self): return True else: if self.mod_short_name is None: - self.log.error("Toolchain module name was not set yet (using set_module_info).") + raise EasyBuildError("Toolchain module name was not set yet (using set_module_info).") # check whether a matching module exists if self.mod_short_name contains a module name return self.modules_tool.exist([self.mod_full_name])[0] @@ -247,7 +250,7 @@ def set_options(self, options): else: # used to be warning, but this is a severe error imho known_opts = ','.join(self.options.keys()) - self.log.error("Undefined toolchain option %s specified (known options: %s)" % (opt, known_opts)) + raise EasyBuildError("Undefined toolchain option %s specified (known options: %s)", opt, known_opts) def get_dependency_version(self, dependency): """ Generate a version string for a dependency on a module using this toolchain """ @@ -267,7 +270,7 @@ def get_dependency_version(self, dependency): if 'version' in dependency: version = "".join([dependency['version'], toolchain, suffix]) - self.log.debug("get_dependency_version: version in dependency return %s" % version) + self.log.debug("get_dependency_version: version in dependency return %s", version) return version else: toolchain_suffix = "".join([toolchain, suffix]) @@ -275,11 +278,11 @@ def get_dependency_version(self, dependency): # Find the most recent (or default) one if len(matches) > 0: version = matches[-1][-1] - self.log.debug("get_dependency_version: version not in dependency return %s" % version) + self.log.debug("get_dependency_version: version not in dependency return %s", version) return else: - tup = (dependency['name'], toolchain_suffix) - self.log.error("No toolchain version for dependency name %s (suffix %s) found" % tup) + raise EasyBuildError("No toolchain version for dependency name %s (suffix %s) found", + dependency['name'], toolchain_suffix) def add_dependencies(self, dependencies): """ Verify if the given dependencies exist and add them """ @@ -289,8 +292,7 @@ def add_dependencies(self, dependencies): for dep, dep_mod_name, dep_exists in zip(dependencies, dep_mod_names, deps_exist): self.log.debug("add_dependencies: MODULEPATH: %s" % os.environ['MODULEPATH']) if not dep_exists: - tup = (dep_mod_name, dep) - self.log.error("add_dependencies: no module '%s' found for dependency %s" % tup) + raise EasyBuildError("add_dependencies: no module '%s' found for dependency %s", dep_mod_name, dep) else: self.dependencies.append(dep) self.log.debug('add_dependencies: added toolchain dependency %s' % str(dep)) @@ -317,6 +319,45 @@ def is_dep_in_toolchain_module(self, name): """Check whether a specific software name is listed as a dependency in the module for this toolchain.""" return any(map(lambda m: self.mns.is_short_modname_for(m, name), self.toolchain_dep_mods)) + def _prepare_dependency_external_module(self, dep): + """Set environment variables picked up by utility functions for dependencies specified as external modules.""" + mod_name = dep['full_mod_name'] + metadata = dep['external_module_metadata'] + self.log.debug("Defining $EB* environment variables for external module %s", mod_name) + + names = metadata.get('name', []) + versions = metadata.get('version', [None]*len(names)) + self.log.debug("Metadata for external module %s: %s", mod_name, metadata) + + for name, version in zip(names, versions): + self.log.debug("Defining $EB* environment variables for external module %s under name %s", mod_name, name) + + # define $EBROOT env var for install prefix, picked up by get_software_root + prefix = metadata.get('prefix') + if prefix is not None: + if prefix in os.environ: + val = os.environ[prefix] + self.log.debug("Using value of $%s as prefix for external module %s: %s", prefix, mod_name, val) + else: + val = prefix + self.log.debug("Using specified prefix for external module %s: %s", mod_name, val) + setvar(get_software_root_env_var_name(name), val) + + # define $EBVERSION env var for software version, picked up by get_software_version + if version is not None: + setvar(get_software_version_env_var_name(name), version) + + def _prepare_dependencies(self): + """Load modules for dependencies, and handle special cases like external modules.""" + # load modules for all dependencies + dep_mods = [dep['short_mod_name'] for dep in self.dependencies] + self.log.debug("Loading modules for dependencies: %s" % dep_mods) + self.modules_tool.load(dep_mods) + + # define $EBROOT* and $EBVERSION* for external modules, if metadata is available + for dep in [d for d in self.dependencies if d['external_module']]: + self._prepare_dependency_external_module(dep) + def prepare(self, onlymod=None): """ Prepare a set of environment parameters based on name/version of toolchain @@ -328,17 +369,17 @@ def prepare(self, onlymod=None): (If string: comma separated list of variables that will be ignored). """ if self.modules_tool is None: - self.log.error("No modules tool defined in Toolchain instance.") + raise EasyBuildError("No modules tool defined in Toolchain instance.") if not self._toolchain_exists(): - self.log.error("No module found for toolchain: %s" % self.mod_short_name) + raise EasyBuildError("No module found for toolchain: %s", self.mod_short_name) if self.name == DUMMY_TOOLCHAIN_NAME: if self.version == DUMMY_TOOLCHAIN_VERSION: self.log.info('prepare: toolchain dummy mode, dummy version; not loading dependencies') else: self.log.info('prepare: toolchain dummy mode and loading dependencies') - self.modules_tool.load([dep['short_mod_name'] for dep in self.dependencies]) + self._prepare_dependencies() return # Load the toolchain and dependencies modules @@ -349,11 +390,11 @@ def prepare(self, onlymod=None): for modpath in self.init_modpaths: self.modules_tool.prepend_module_path(os.path.join(install_path('mod'), mod_path_suffix, modpath)) self.modules_tool.load([self.det_short_module_name()]) - self.modules_tool.load([dep['short_mod_name'] for dep in self.dependencies]) + self._prepare_dependencies() # determine direct toolchain dependencies mod_name = self.det_short_module_name() - self.toolchain_dep_mods = self.modules_tool.dependencies_for(mod_name, depth=0) + self.toolchain_dep_mods = dependencies_for(mod_name, depth=0) self.log.debug('prepare: list of direct toolchain dependencies: %s' % self.toolchain_dep_mods) # only retain names of toolchain elements, excluding toolchain name @@ -373,8 +414,8 @@ def prepare(self, onlymod=None): if all(map(self.is_dep_in_toolchain_module, toolchain_definition)): self.log.info("List of toolchain dependency modules and toolchain definition match!") else: - self.log.error("List of toolchain dependency modules and toolchain definition do not match " \ - "(%s vs %s)" % (self.toolchain_dep_mods, toolchain_definition)) + raise EasyBuildError("List of toolchain dependency modules and toolchain definition do not match " + "(%s vs %s)", self.toolchain_dep_mods, toolchain_definition) # Generate the variables to be set self.set_variables() @@ -415,7 +456,17 @@ def _add_dependency_variables(self, names=None, cpp=None, ld=None): else: deps = [{'name': name} for name in names if name is not None] - for root in self.get_software_root([dep['name'] for dep in deps]): + # collect software install prefixes for dependencies + roots = [] + for dep in deps: + if dep.get('external_module', False): + # for software names provided via external modules, install prefix may be unknown + names = dep['external_module_metadata'].get('name', []) + roots.extend([root for root in self.get_software_root(names) if root is not None]) + else: + roots.extend(self.get_software_root(dep['name'])) + + for root in roots: self.variables.append_subdirs("CPPFLAGS", root, subdirs=cpp_paths) self.variables.append_subdirs("LDFLAGS", root, subdirs=ld_paths) @@ -426,8 +477,7 @@ def _setenv_variables(self, donotset=None): donotsetlist = [] if isinstance(donotset, str): # TODO : more legacy code that should be using proper type - self.log.error("_setenv_variables: using commas-separated list. should be deprecated.") - donotsetlist = donotset.split(',') + raise EasyBuildError("_setenv_variables: using commas-separated list. should be deprecated.") elif isinstance(donotset, list): donotsetlist = donotset @@ -449,6 +499,10 @@ def get_flag(self, name): """Get compiler flag for a certain option.""" return "-%s" % self.options.option(name) + def toolchain_family(self): + """Return toolchain family for this toolchain.""" + return self.TOOLCHAIN_FAMILY + def comp_family(self): """ Return compiler family used in this toolchain (abstract method).""" raise NotImplementedError diff --git a/easybuild/tools/toolchain/toolchainvariables.py b/easybuild/tools/toolchain/toolchainvariables.py index 0c94c8874c..a9c4682152 100644 --- a/easybuild/tools/toolchain/toolchainvariables.py +++ b/easybuild/tools/toolchain/toolchainvariables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/utilities.py b/easybuild/tools/toolchain/utilities.py index 633e10accb..73bf78c7c1 100644 --- a/easybuild/tools/toolchain/utilities.py +++ b/easybuild/tools/toolchain/utilities.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -40,6 +40,7 @@ from vsc.utils.missing import get_subclasses, nub import easybuild.tools.toolchain +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.toolchain import Toolchain from easybuild.tools.utilities import import_available_modules @@ -83,13 +84,13 @@ def search_toolchain(name): if res: tc_const_name = res.group(1) tc_const_value = getattr(mod_class_mod, elem) - tup = (tc_const_name, tc_const_value, mod_class_mod.__name__, package.__name__) - _log.debug("Found constant %s ('%s') in module %s, adding it to %s" % tup) + _log.debug("Found constant %s ('%s') in module %s, adding it to %s", + tc_const_name, tc_const_value, mod_class_mod.__name__, package.__name__) if hasattr(package, tc_const_name): cur_value = getattr(package, tc_const_name) if not tc_const_value == cur_value: - tup = (package.__name__, tc_const_name, cur_value, tc_const_value) - _log.error("Constant %s.%s defined as '%s', can't set it to '%s'." % tup) + raise EasyBuildError("Constant %s.%s defined as '%s', can't set it to '%s'.", + package.__name__, tc_const_name, cur_value, tc_const_value) else: setattr(package, tc_const_name, tc_const_value) @@ -124,7 +125,7 @@ def get_toolchain(tc, tcopts, mns): tc_class, all_tcs = search_toolchain(tc['name']) if not tc_class: all_tcs_names = ",".join([x.NAME for x in all_tcs]) - _log.error("Toolchain %s not found, available toolchains: %s" % (tc['name'], all_tcs_names)) + raise EasyBuildError("Toolchain %s not found, available toolchains: %s", tc['name'], all_tcs_names) tc_inst = tc_class(version=tc['version'], mns=mns) tc_dict = tc_inst.as_dict() _log.debug("Obtained new toolchain instance for %s: %s" % (key, tc_dict)) diff --git a/easybuild/tools/toolchain/variables.py b/easybuild/tools/toolchain/variables.py index b9fb56d00e..cdee3ea41b 100644 --- a/easybuild/tools/toolchain/variables.py +++ b/easybuild/tools/toolchain/variables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -29,6 +29,7 @@ @author: Kenneth Hoste (Ghent University) """ +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.variables import StrList, AbsPathList @@ -141,7 +142,7 @@ def _toggle_map(self, toggle_map, name, descr, idx=None): else: self.insert(idx, toggle_map[name]) else: - self.log.raiseException("%s name %s not found in map %s" % (descr, name, toggle_map)) + raise EasyBuildError("%s name %s not found in map %s", descr, name, toggle_map) def toggle_startgroup(self): """Append start group""" diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index 096766e127..fb3ac98fe0 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/variables.py b/easybuild/tools/variables.py index bee298f6ce..a6ec60c62f 100644 --- a/easybuild/tools/variables.py +++ b/easybuild/tools/variables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -29,9 +29,12 @@ @author: Stijn De Weirdt (Ghent University) @author: Kenneth Hoste (Ghent University) """ -from vsc.utils import fancylogger import copy import os +from vsc.utils import fancylogger + +from easybuild.tools.build_log import EasyBuildError + _log = fancylogger.getLogger('variables', fname=False) @@ -62,20 +65,20 @@ def join_map_class(map_classes): """Join all class_maps into single class_map""" res = {} for map_class in map_classes: - for k, v in map_class.items(): - if isinstance(k, (str,)): - var_name = k - if isinstance(v, (tuple, list)): + for key, val in map_class.items(): + if isinstance(key, (str,)): + var_name = key + if isinstance(val, (tuple, list)): # second element is documentation - klass = v[0] + klass = val[0] res[var_name] = klass - elif type(k) in (type,): + elif type(key) in (type,): # k is the class, v a list of tuples (name,doc) - klass = k + klass = key default = res.setdefault(klass, []) - default.extend([tpl[0] for tpl in v]) + default.extend([tpl[0] for tpl in val]) else: - _log.raiseException("join_map_class: impossible to join key %s value %s" % (k, v)) + raise EasyBuildError("join_map_class: impossible to join key %s value %s", key, val) return res @@ -304,7 +307,7 @@ def nextend(self, value=None, **kwargs): res = [] if value is None: # TODO ? append_empty ? - self.log.raiseException("extend_el with None value unimplemented") + raise EasyBuildError("extend_el with None value unimplemented") else: for el in value: if not self._str_ok(el): @@ -478,13 +481,17 @@ def join(self, name, *others): else it is nappend-ed """ self.log.debug("join name %s others %s" % (name, others)) + + # make sure name is defined, even if 'others' list is empty + self.setdefault(name) + for other in others: if other in self: self.log.debug("join other %s in self: other %s" % (other, self.get(other).__repr__())) for el in self.get(other): self.nappend(name, el) else: - self.log.raiseException("join: name %s; other %s not found in self." % (name, other)) + raise EasyBuildError("join: name %s; other %s not found in self.", name, other) def append(self, name, value): """Append value to element name (alias for nappend)""" diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 1bd1419af5..2efb989a88 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,8 +37,8 @@ # note: release candidates should be versioned as a pre-release, e.g. "1.1rc1" # 1.1-rc1 would indicate a post-release, i.e., and update of 1.1, so beware! -VERSION = LooseVersion("2.0.0dev") -UNKNOWN = "UNKNOWN" +VERSION = LooseVersion('2.2.0dev') +UNKNOWN = 'UNKNOWN' def get_git_revision(): """ diff --git a/setup.py b/setup.py index fcb6f4b2dd..edf1ff7655 100644 --- a/setup.py +++ b/setup.py @@ -106,5 +106,5 @@ def find_rel_test(): provides=["eb"] + easybuild_packages, test_suite="test.framework.suite", zip_safe=False, - install_requires=["vsc-base >= 2.0.3"], + install_requires=["vsc-base >= 2.2.0"], ) diff --git a/test/__init__.py b/test/__init__.py index 9459a6d90a..8df00ef461 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/__init__.py b/test/framework/__init__.py index f7e235f8d2..d6e0a952fc 100644 --- a/test/framework/__init__.py +++ b/test/framework/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/asyncprocess.py b/test/framework/asyncprocess.py index c70b4b1dd4..d8637fecf5 100644 --- a/test/framework/asyncprocess.py +++ b/test/framework/asyncprocess.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/build_log.py b/test/framework/build_log.py new file mode 100644 index 0000000000..814308ba87 --- /dev/null +++ b/test/framework/build_log.py @@ -0,0 +1,152 @@ +# # +# Copyright 2015-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Unit tests for EasyBuild log infrastructure + +@author: Kenneth Hoste (Ghent University) +""" +import os +import re +import tempfile +from test.framework.utilities import EnhancedTestCase, init_config +from unittest import TestLoader +from unittest import main as unittestmain +from vsc.utils.fancylogger import getLogger, getRootLoggerName, logToFile, setLogFormat + +from easybuild.tools.build_log import LOGGING_FORMAT, EasyBuildError +from easybuild.tools.filetools import read_file, write_file + + +def raise_easybuilderror(msg, *args, **kwargs): + """Utility function: just raise a EasyBuildError.""" + raise EasyBuildError(msg, *args, **kwargs) + + +class BuildLogTest(EnhancedTestCase): + """Tests for EasyBuild log infrastructure.""" + + def tearDown(self): + """Cleanup after test.""" + # restore default logging format + setLogFormat(LOGGING_FORMAT) + + def test_easybuilderror(self): + """Tests for EasyBuildError.""" + fd, tmplog = tempfile.mkstemp() + os.close(fd) + + # set log format, for each regex searching + setLogFormat("%(name)s :: %(message)s") + + # if no logger is available, and no logger is specified, use default 'root' fancylogger + logToFile(tmplog, enable=True) + self.assertErrorRegex(EasyBuildError, 'BOOM', raise_easybuilderror, 'BOOM') + logToFile(tmplog, enable=False) + + log_re = re.compile("^%s :: BOOM \(at .*:[0-9]+ in [a-z_]+\)$" % getRootLoggerName(), re.M) + logtxt = open(tmplog, 'r').read() + self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt)) + + # test formatting of message + self.assertErrorRegex(EasyBuildError, 'BOOMBAF', raise_easybuilderror, 'BOOM%s', 'BAF') + + os.remove(tmplog) + + def test_easybuildlog(self): + """Tests for EasyBuildLog.""" + fd, tmplog = tempfile.mkstemp() + os.close(fd) + + # set log format, for each regex searching + setLogFormat("%(name)s [%(levelname)s] :: %(message)s") + + # test basic log methods + logToFile(tmplog, enable=True) + log = getLogger('test_easybuildlog') + log.setLevelName('DEBUG') + log.debug("123 debug") + log.info("foobar info") + log.warn("justawarning") + log.raiseError = False + log.error("kaput") + log.raiseError = True + try: + log.exception("oops") + except EasyBuildError: + pass + logToFile(tmplog, enable=False) + logtxt = read_file(tmplog) + + root = getRootLoggerName() + + expected_logtxt = '\n'.join([ + r"%s.test_easybuildlog \[DEBUG\] :: 123 debug" % root, + r"%s.test_easybuildlog \[INFO\] :: foobar info" % root, + r"%s.test_easybuildlog \[WARNING\] :: justawarning" % root, + r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput" % root, + r"%s.test_easybuildlog \[ERROR\] :: .*EasyBuild encountered an exception \(at .* in .*\): oops" % root, + '', + ]) + logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M) + self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt)) + + # wipe log so we can reuse it + write_file(tmplog, '') + + # test formatting log messages by providing extra arguments + logToFile(tmplog, enable=True) + log.warn("%s", "bleh"), + log.info("%s+%s = %d", '4', '2', 42) + args = ['this', 'is', 'just', 'a', 'test'] + log.debug("%s %s %s %s %s", *args) + log.raiseError = False + log.error("foo %s baz", 'baz') + log.raiseError = True + logToFile(tmplog, enable=False) + logtxt = read_file(tmplog) + expected_logtxt = '\n'.join([ + r"%s.test_easybuildlog \[WARNING\] :: bleh" % root, + r"%s.test_easybuildlog \[INFO\] :: 4\+2 = 42" % root, + r"%s.test_easybuildlog \[DEBUG\] :: this is just a test" % root, + r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): foo baz baz" % root, + '', + ]) + logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M) + self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt)) + + # test deprecated behaviour: raise EasyBuildError on log.error and log.exception + os.environ['EASYBUILD_DEPRECATED'] = '2.1' + init_config() + + log.warning("No raise for warnings") + self.assertErrorRegex(EasyBuildError, 'EasyBuild crashed with an error', log.error, 'foo') + self.assertErrorRegex(EasyBuildError, 'EasyBuild encountered an exception', log.exception, 'bar') + +def suite(): + """ returns all the testcases in this module """ + return TestLoader().loadTestsFromTestCase(BuildLogTest) + +if __name__ == '__main__': + unittestmain() diff --git a/test/framework/config.py b/test/framework/config.py index 36de96ad16..ecbb28fe2c 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -38,14 +38,33 @@ from vsc.utils.fancylogger import setLogLevelDebug, logToScreen import easybuild.tools.options as eboptions -from easybuild.tools.config import build_path, source_paths, install_path, get_repositorypath +from easybuild.tools import run +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import build_option, build_path, source_paths, install_path, get_repositorypath from easybuild.tools.config import set_tmpdir, BuildOptions, ConfigurationVariables -from easybuild.tools.config import get_build_log_path, DEFAULT_PATH_SUBDIRS, init_build_options, build_option +from easybuild.tools.config import get_build_log_path, DEFAULT_PATH_SUBDIRS, init_build_options from easybuild.tools.environment import modify_env from easybuild.tools.filetools import mkdir, write_file -from easybuild.tools.repository.filerepo import FileRepository -from easybuild.tools.repository.repository import init_repository - +from easybuild.tools.options import CONFIG_ENV_VAR_PREFIX + +EXTERNAL_MODULES_METADATA = """[cray-netcdf/4.3.2] +name = netCDF,netCDF-Fortran +version = 4.3.2,4.3.2 +prefix = NETCDF_DIR + +[cray-hdf5/1.8.13] +name = HDF5 +version = 1.8.13 +prefix = HDF5_DIR + +[foo] +name = Foo +prefix = /foo + +[bar/1.2.3] +name = bar +version = 1.2.3 +""" class EasyBuildConfigTest(EnhancedTestCase): """Test cases for EasyBuild configuration.""" @@ -54,6 +73,7 @@ class EasyBuildConfigTest(EnhancedTestCase): def setUp(self): """Prepare for running a config test.""" + reload(eboptions) super(EasyBuildConfigTest, self).setUp() self.tmpdir = tempfile.mkdtemp() @@ -137,12 +157,13 @@ def test_generaloption_config(self): '--prefix', prefix, '--installpath', install, '--repositorypath', repopath, + '--subdir-software', 'APPS', ] options = init_config(args=args) self.assertEqual(build_path(), os.path.join(prefix, 'build')) - self.assertEqual(install_path(), os.path.join(install, 'software')) + self.assertEqual(install_path(), os.path.join(install, 'APPS')) self.assertEqual(install_path(typ='mod'), os.path.join(install, 'modules')) self.assertEqual(options.installpath, install) @@ -159,27 +180,83 @@ def test_generaloption_config(self): os.environ['EASYBUILD_PREFIX'] = prefix os.environ['EASYBUILD_SUBDIR_SOFTWARE'] = subdir_software + installpath_modules = tempfile.mkdtemp(prefix='installpath-modules') + os.environ['EASYBUILD_INSTALLPATH_MODULES'] = installpath_modules options = init_config(args=args) self.assertEqual(build_path(), os.path.join(prefix, 'build')) self.assertEqual(install_path(), os.path.join(install, subdir_software)) + self.assertEqual(install_path('mod'), installpath_modules) + + # subdir options *must* be relative (to --installpath) + installpath_software = tempfile.mkdtemp(prefix='installpath-software') + os.environ['EASYBUILD_SUBDIR_SOFTWARE'] = installpath_software + error_regex = r"Found problems validating the options.*'subdir_software' must specify a \*relative\* path" + self.assertErrorRegex(EasyBuildError, error_regex, init_config) del os.environ['EASYBUILD_PREFIX'] del os.environ['EASYBUILD_SUBDIR_SOFTWARE'] + def test_error_env_var_typo(self): + """Test error reporting on use of known $EASYBUILD-prefixed env vars.""" + # all is well + init_config() + + os.environ['EASYBUILD_FOO'] = 'foo' + os.environ['EASYBUILD_THERESNOSUCHCONFIGURATIONOPTION'] = 'whatever' + + error = r"Found 2 environment variable\(s\) that are prefixed with %s " % CONFIG_ENV_VAR_PREFIX + error += "but do not match valid option\(s\): " + error += ','.join(['EASYBUILD_FOO', 'EASYBUILD_THERESNOSUCHCONFIGURATIONOPTION']) + self.assertErrorRegex(EasyBuildError, error, init_config) + + del os.environ['EASYBUILD_THERESNOSUCHCONFIGURATIONOPTION'] + del os.environ['EASYBUILD_FOO'] + + def test_install_path(self): + """Test install_path function.""" + # defaults + self.assertEqual(install_path(), os.path.join(self.test_installpath, 'software')) + self.assertEqual(install_path('software'), os.path.join(self.test_installpath, 'software')) + self.assertEqual(install_path(typ='mod'), os.path.join(self.test_installpath, 'modules')) + self.assertEqual(install_path('modules'), os.path.join(self.test_installpath, 'modules')) + + self.assertErrorRegex(EasyBuildError, "Unknown type specified", install_path, typ='foo') + + args = [ + '--subdir-software', 'SOFT', + '--installpath', '/foo', + ] + os.environ['EASYBUILD_SUBDIR_MODULES'] = 'MOD' + init_config(args=args) + self.assertEqual(install_path(), os.path.join('/foo', 'SOFT')) + self.assertEqual(install_path(typ='mod'), os.path.join('/foo', 'MOD')) + del os.environ['EASYBUILD_SUBDIR_MODULES'] + + args = [ + '--installpath', '/prefix', + '--installpath-modules', '/foo', + ] + os.environ['EASYBUILD_INSTALLPATH_SOFTWARE'] = '/bar/baz' + init_config(args=args) + self.assertEqual(install_path(), os.path.join('/bar', 'baz')) + self.assertEqual(install_path(typ='mod'), '/foo') + + del os.environ['EASYBUILD_INSTALLPATH_SOFTWARE'] + init_config(args=args) + self.assertEqual(install_path(), os.path.join('/prefix', 'software')) + self.assertEqual(install_path(typ='mod'), '/foo') + def test_generaloption_config_file(self): """Test use of new-style configuration file.""" self.purge_environment() - oldstyle_config_file = os.path.join(self.tmpdir, 'nooldconfig.py') config_file = os.path.join(self.tmpdir, 'testconfig.cfg') testpath1 = os.path.join(self.tmpdir, 'test1') testpath2 = os.path.join(self.tmpdir, 'testtwo') - write_file(oldstyle_config_file, '') - # test with config file passed via command line cfgtxt = '\n'.join([ '[config]', @@ -187,16 +264,19 @@ def test_generaloption_config_file(self): ]) write_file(config_file, cfgtxt) + installpath_software = tempfile.mkdtemp(prefix='installpath-software') args = [ '--configfiles', config_file, '--debug', '--buildpath', testpath1, + '--installpath-software', installpath_software, ] options = init_config(args=args) self.assertEqual(build_path(), testpath1) # via command line self.assertEqual(source_paths(), [os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'sources')]) # default - self.assertEqual(install_path(), os.path.join(testpath2, 'software')) # via config file + self.assertEqual(install_path(), installpath_software) # via cmdline arg + self.assertEqual(install_path('mod'), os.path.join(testpath2, 'modules')) # via config file # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory # to check whether easyconfigs install path is auto-included in robot path @@ -210,10 +290,14 @@ def test_generaloption_config_file(self): sys.path.insert(0, tmpdir) # prepend to give it preference over possible other installed easyconfigs pkgs # test with config file passed via environment variable + installpath_modules = tempfile.mkdtemp(prefix='installpath-modules') cfgtxt = '\n'.join([ '[config]', 'buildpath = %s' % testpath1, - 'robot-paths = /tmp/foo:%(DEFAULT_ROBOT_PATHS)s', + 'sourcepath = %(DEFAULT_REPOSITORYPATH)s', + 'repositorypath = %(DEFAULT_REPOSITORYPATH)s,somesubdir', + 'robot-paths=/tmp/foo:%(sourcepath)s:%(DEFAULT_ROBOT_PATHS)s', + 'installpath-modules=%s' % installpath_modules, ]) write_file(config_file, cfgtxt) @@ -224,11 +308,18 @@ def test_generaloption_config_file(self): ] options = init_config(args=args) - self.assertEqual(install_path(), os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'software')) # default + topdir = os.path.join(os.getenv('HOME'), '.local', 'easybuild') + self.assertEqual(install_path(), os.path.join(topdir, 'software')) # default + self.assertEqual(install_path('mod'), installpath_modules), # via config file self.assertEqual(source_paths(), [testpath2]) # via command line self.assertEqual(build_path(), testpath1) # via config file - self.assertTrue('/tmp/foo' in options.robot_paths) - self.assertTrue(os.path.join(tmpdir, 'easybuild', 'easyconfigs') in options.robot_paths) + self.assertEqual(get_repositorypath(), [os.path.join(topdir, 'ebfiles_repo'), 'somesubdir']) # via config file + robot_paths = [ + '/tmp/foo', + os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'ebfiles_repo'), + os.path.join(tmpdir, 'easybuild', 'easyconfigs'), + ] + self.assertEqual(options.robot_paths[:3], robot_paths) testpath3 = os.path.join(self.tmpdir, 'testTHREE') os.environ['EASYBUILD_SOURCEPATH'] = testpath2 @@ -240,6 +331,7 @@ def test_generaloption_config_file(self): self.assertEqual(source_paths(), [testpath2]) # via environment variable $EASYBUILD_SOURCEPATHS self.assertEqual(install_path(), os.path.join(testpath3, 'software')) # via command line + self.assertEqual(install_path('mod'), installpath_modules), # via config file self.assertEqual(build_path(), testpath1) # via config file del os.environ['EASYBUILD_CONFIGFILES'] @@ -257,13 +349,13 @@ def test_set_tmpdir(self): mytmpdir = set_tmpdir(tmpdir=tmpdir) for var in ['TMPDIR', 'TEMP', 'TMP']: - self.assertTrue(os.environ[var].startswith(os.path.join(parent, 'easybuild-'))) + self.assertTrue(os.environ[var].startswith(os.path.join(parent, 'eb-'))) self.assertEqual(os.environ[var], mytmpdir) - self.assertTrue(tempfile.gettempdir().startswith(os.path.join(parent, 'easybuild-'))) + self.assertTrue(tempfile.gettempdir().startswith(os.path.join(parent, 'eb-'))) tempfile_tmpdir = tempfile.mkdtemp() - self.assertTrue(tempfile_tmpdir.startswith(os.path.join(parent, 'easybuild-'))) + self.assertTrue(tempfile_tmpdir.startswith(os.path.join(parent, 'eb-'))) fd, tempfile_tmpfile = tempfile.mkstemp() - self.assertTrue(tempfile_tmpfile.startswith(os.path.join(parent, 'easybuild-'))) + self.assertTrue(tempfile_tmpfile.startswith(os.path.join(parent, 'eb-'))) # tmp_logdir follows tmpdir self.assertEqual(get_build_log_path(), mytmpdir) @@ -389,12 +481,11 @@ def test_XDG_CONFIG_env_vars(self): os.path.join(dir1, 'easybuild.d', 'bar.cfg'), os.path.join(dir1, 'easybuild.d', 'foo.cfg'), os.path.join(dir3, 'easybuild.d', 'foobarbaz.cfg'), - # default config file in home dir is last (even if the file is not there) - os.path.join(os.path.expanduser('~'), '.config', 'easybuild', 'config.cfg'), ] reload(eboptions) eb_go = eboptions.parse_options(args=[]) - self.assertEqual(eb_go.options.configfiles, cfg_files) + # note: there may be a config file in $HOME too, so don't use a strict comparison + self.assertEqual(cfg_files, eb_go.options.configfiles[:3]) # $XDG_CONFIG_HOME set to non-existing directory, multiple directories listed in $XDG_CONFIG_DIRS os.environ['XDG_CONFIG_HOME'] = os.path.join(self.test_prefix, 'nosuchdir') @@ -402,7 +493,6 @@ def test_XDG_CONFIG_env_vars(self): os.path.join(dir1, 'easybuild.d', 'bar.cfg'), os.path.join(dir1, 'easybuild.d', 'foo.cfg'), os.path.join(dir3, 'easybuild.d', 'foobarbaz.cfg'), - os.path.join(self.test_prefix, 'nosuchdir', 'easybuild', 'config.cfg'), ] reload(eboptions) eb_go = eboptions.parse_options(args=[]) @@ -419,6 +509,142 @@ def test_XDG_CONFIG_env_vars(self): del os.environ['XDG_CONFIG_DIRS'] else: os.environ['XDG_CONFIG_DIRS'] = xdg_config_dirs + reload(eboptions) + + def test_flex_robot_paths(self): + """Test prepend/appending to default robot search path via --robot-paths.""" + # unset $EASYBUILD_ROBOT_PATHS that was defined in setUp + del os.environ['EASYBUILD_ROBOT_PATHS'] + + # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory + # to check whether easyconfigs install path is auto-included in robot path + tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') + mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) + test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + tmp_ecs_dir = os.path.join(tmpdir, 'easybuild', 'easyconfigs') + shutil.copytree(test_ecs_path, tmp_ecs_dir) + + # prepend path to test easyconfigs into Python search path, so it gets picked up as --robot-paths default + orig_sys_path = sys.path[:] + sys.path = [tmpdir] + [p for p in sys.path if not os.path.exists(os.path.join(p, 'easybuild', 'easyconfigs'))] + + # default: only pick up installed easyconfigs via sys.path + eb_go = eboptions.parse_options(args=[]) + self.assertEqual(eb_go.options.robot_paths, [tmp_ecs_dir]) + + # prepend to default robot path + eb_go = eboptions.parse_options(args=['--robot-paths=/foo:']) + self.assertEqual(eb_go.options.robot_paths, ['/foo', tmp_ecs_dir]) + eb_go = eboptions.parse_options(args=['--robot-paths=/foo:/bar/baz/:']) + self.assertEqual(eb_go.options.robot_paths, ['/foo', '/bar/baz/', tmp_ecs_dir]) + + # append to default robot path + eb_go = eboptions.parse_options(args=['--robot-paths=:/bar/baz']) + self.assertEqual(eb_go.options.robot_paths, [tmp_ecs_dir, '/bar/baz']) + # append to default robot path + eb_go = eboptions.parse_options(args=['--robot-paths=:/bar/baz:/foo']) + self.assertEqual(eb_go.options.robot_paths, [tmp_ecs_dir, '/bar/baz', '/foo']) + + # prepend and append to default robot path + eb_go = eboptions.parse_options(args=['--robot-paths=/foo/bar::/baz']) + self.assertEqual(eb_go.options.robot_paths, ['/foo/bar', tmp_ecs_dir, '/baz']) + eb_go = eboptions.parse_options(args=['--robot-paths=/foo/bar::/baz:/trala']) + self.assertEqual(eb_go.options.robot_paths, ['/foo/bar', tmp_ecs_dir, '/baz', '/trala']) + eb_go = eboptions.parse_options(args=['--robot-paths=/foo/bar:/trala::/baz']) + self.assertEqual(eb_go.options.robot_paths, ['/foo/bar', '/trala', tmp_ecs_dir, '/baz']) + + # also via $EASYBUILD_ROBOT_PATHS + os.environ['EASYBUILD_ROBOT_PATHS'] = '/foo::/bar/baz' + eb_go = eboptions.parse_options(args=[]) + self.assertEqual(eb_go.options.robot_paths, ['/foo', tmp_ecs_dir, '/bar/baz']) + + # --robot-paths overrides $EASYBUILD_ROBOT_PATHS + os.environ['EASYBUILD_ROBOT_PATHS'] = '/foobar::/barbar/baz/baz' + eb_go = eboptions.parse_options(args=['--robot-paths=/one::/last']) + self.assertEqual(eb_go.options.robot_paths, ['/one', tmp_ecs_dir, '/last']) + + del os.environ['EASYBUILD_ROBOT_PATHS'] + + # also works with a cfgfile in the mix + config_file = os.path.join(self.tmpdir, 'testconfig.cfg') + cfgtxt = '\n'.join([ + '[config]', + 'robot-paths=/cfgfirst::/cfglast', + ]) + write_file(config_file, cfgtxt) + eb_go = eboptions.parse_options(args=['--configfiles=%s' % config_file]) + self.assertEqual(eb_go.options.robot_paths, ['/cfgfirst', tmp_ecs_dir, '/cfglast']) + + # cfgfile entry is lost when env var and/or cmdline options are used + os.environ['EASYBUILD_ROBOT_PATHS'] = '/envfirst::/envend' + eb_go = eboptions.parse_options(args=['--configfiles=%s' % config_file]) + self.assertEqual(eb_go.options.robot_paths, ['/envfirst', tmp_ecs_dir, '/envend']) + + del os.environ['EASYBUILD_ROBOT_PATHS'] + eb_go = eboptions.parse_options(args=['--robot-paths=/veryfirst:', '--configfiles=%s' % config_file]) + self.assertEqual(eb_go.options.robot_paths, ['/veryfirst', tmp_ecs_dir]) + + os.environ['EASYBUILD_ROBOT_PATHS'] = ':/envend' + eb_go = eboptions.parse_options(args=['--robot-paths=/veryfirst:', '--configfiles=%s' % config_file]) + self.assertEqual(eb_go.options.robot_paths, ['/veryfirst', tmp_ecs_dir]) + + del os.environ['EASYBUILD_ROBOT_PATHS'] + + # override default robot path + eb_go = eboptions.parse_options(args=['--robot-paths=/foo:/bar/baz']) + self.assertEqual(eb_go.options.robot_paths, ['/foo', '/bar/baz']) + + # paths specified via --robot still get preference + eb_go = eboptions.parse_options(args=['--robot-paths=/foo/bar::/baz', '--robot=/first']) + self.assertEqual(eb_go.options.robot_paths, ['/first', '/foo/bar', tmp_ecs_dir, '/baz']) + + sys.path[:] = orig_sys_path + + def test_external_modules_metadata(self): + """Test --external-modules-metadata.""" + # empty list by default + cfg = init_config() + self.assertEqual(cfg.external_modules_metadata, []) + + testcfgtxt = EXTERNAL_MODULES_METADATA + testcfg = os.path.join(self.test_prefix, 'test_external_modules_metadata.cfg') + write_file(testcfg, testcfgtxt) + + cfg = init_config(args=['--external-modules-metadata=%s' % testcfg]) + + netcdf = { + 'name': ['netCDF', 'netCDF-Fortran'], + 'version': ['4.3.2', '4.3.2'], + 'prefix': 'NETCDF_DIR', + } + self.assertEqual(cfg.external_modules_metadata['cray-netcdf/4.3.2'], netcdf) + hdf5 = { + 'name': ['HDF5'], + 'version': ['1.8.13'], + 'prefix': 'HDF5_DIR', + } + self.assertEqual(cfg.external_modules_metadata['cray-hdf5/1.8.13'], hdf5) + + # impartial metadata is fine + self.assertEqual(cfg.external_modules_metadata['foo'], {'name': ['Foo'], 'prefix': '/foo'}) + self.assertEqual(cfg.external_modules_metadata['bar/1.2.3'], {'name': ['bar'], 'version': ['1.2.3']}) + + # if both names and versions are specified, lists must have same lengths + write_file(testcfg, '\n'.join(['[foo/1.2.3]', 'name = foo,bar', 'version = 1.2.3'])) + args = ['--external-modules-metadata=%s' % testcfg] + err_msg = "Different length for lists of names/versions in metadata for external module" + self.assertErrorRegex(EasyBuildError, err_msg, init_config, args=args) + + def test_strict(self): + """Test use of --strict.""" + # check default + self.assertEqual(build_option('strict'), run.WARN) + + for strict_str, strict_val in [('error', run.ERROR), ('ignore', run.IGNORE), ('warn', run.WARN)]: + options = init_config(args=['--strict=%s' % strict_str]) + init_config(build_options={'strict': options.strict}) + self.assertEqual(build_option('strict'), strict_val) + def suite(): return TestLoader().loadTestsFromTestCase(EasyBuildConfigTest) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 890cc86f28..98fb54d10f 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -43,6 +43,7 @@ from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools import config from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import get_module_syntax from easybuild.tools.filetools import mkdir, read_file, write_file from easybuild.tools.modules import modules_tool @@ -61,7 +62,6 @@ def setUp(self): fd, self.eb_file = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb') os.close(fd) - self.orig_tmp_logdir = os.environ.get('EASYBUILD_TMP_LOGDIR', None) self.test_tmp_logdir = tempfile.mkdtemp() os.environ['EASYBUILD_TMP_LOGDIR'] = self.test_tmp_logdir @@ -192,11 +192,20 @@ def test_make_module_req(self): guess = eb.make_module_req() - self.assertTrue(re.search("^prepend-path\s+CLASSPATH\s+\$root/bla.jar$", guess, re.M)) - self.assertTrue(re.search("^prepend-path\s+CLASSPATH\s+\$root/foo.jar$", guess, re.M)) - self.assertTrue(re.search("^prepend-path\s+MANPATH\s+\$root/share/man$", guess, re.M)) - self.assertTrue(re.search("^prepend-path\s+PATH\s+\$root/bin$", guess, re.M)) - self.assertFalse(re.search("^prepend-path\s+CPATH\s+.*$", guess, re.M)) + if get_module_syntax() == 'Tcl': + self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/bla.jar$", guess, re.M)) + self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/foo.jar$", guess, re.M)) + self.assertTrue(re.search(r"^prepend-path\s+MANPATH\s+\$root/share/man$", guess, re.M)) + self.assertTrue(re.search(r"^prepend-path\s+PATH\s+\$root/bin$", guess, re.M)) + self.assertFalse(re.search(r"^prepend-path\s+CPATH\s+.*$", guess, re.M)) + elif get_module_syntax() == 'Lua': + self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "bla.jar"\)\)$', guess, re.M)) + self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "foo.jar"\)\)$', guess, re.M)) + self.assertTrue(re.search(r'^prepend_path\("MANPATH", pathJoin\(root, "share/man"\)\)$', guess, re.M)) + self.assertTrue(re.search(r'^prepend_path\("PATH", pathJoin\(root, "bin"\)\)$', guess, re.M)) + self.assertFalse(re.search(r'^prepend_path\("CPATH", .*\)$', guess, re.M)) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) # cleanup eb.close_log() @@ -300,28 +309,63 @@ def test_make_module_step(self): eb.check_readiness_step() modpath = os.path.join(eb.make_module_step(), name, version) + if get_module_syntax() == 'Lua': + modpath += '.lua' self.assertTrue(os.path.exists(modpath), "%s exists" % modpath) # verify contents of module - f = open(modpath, 'r') - txt = f.read() - f.close() - self.assertTrue(re.search("^#%Module", txt.split('\n')[0])) - self.assertTrue(re.search("^conflict\s+%s$" % name, txt, re.M)) - self.assertTrue(re.search("^set\s+root\s+%s$" % eb.installdir, txt, re.M)) - self.assertTrue(re.search('^setenv\s+EBROOT%s\s+".root"\s*$' % name.upper(), txt, re.M)) - self.assertTrue(re.search('^setenv\s+EBVERSION%s\s+"%s"$' % (name.upper(), version), txt, re.M)) + txt = read_file(modpath) + if get_module_syntax() == 'Tcl': + self.assertTrue(re.search(r"^#%Module", txt.split('\n')[0])) + self.assertTrue(re.search(r"^conflict\s+%s$" % name, txt, re.M)) + + self.assertTrue(re.search(r"^set\s+root\s+%s$" % eb.installdir, txt, re.M)) + ebroot_regex = re.compile(r'^setenv\s+EBROOT%s\s+"\$root"\s*$' % name.upper(), re.M) + self.assertTrue(ebroot_regex.search(txt), "%s in %s" % (ebroot_regex.pattern, txt)) + self.assertTrue(re.search(r'^setenv\s+EBVERSION%s\s+"%s"$' % (name.upper(), version), txt, re.M)) + + elif get_module_syntax() == 'Lua': + ebroot_regex = re.compile(r'^setenv\("EBROOT%s", root\)$' % name.upper(), re.M) + self.assertTrue(ebroot_regex.search(txt), "%s in %s" % (ebroot_regex.pattern, txt)) + self.assertTrue(re.search(r'^setenv\("EBVERSION%s", "%s"\)$' % (name.upper(), version), txt, re.M)) + + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + for (key, val) in modextravars.items(): - regex = re.compile('^setenv\s+%s\s+"%s"$' % (key, val), re.M) + if get_module_syntax() == 'Tcl': + regex = re.compile(r'^setenv\s+%s\s+"%s"$' % (key, val), re.M) + elif get_module_syntax() == 'Lua': + regex = re.compile(r'^setenv\("%s", "%s"\)$' % (key, val), re.M) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) + for (key, val) in modextrapaths.items(): - regex = re.compile('^prepend-path\s+%s\s+\$root/%s$' % (key, val), re.M) + if get_module_syntax() == 'Tcl': + regex = re.compile(r'^prepend-path\s+%s\s+\$root/%s$' % (key, val), re.M) + elif get_module_syntax() == 'Lua': + regex = re.compile(r'^prepend_path\("%s", pathJoin\(root, "%s"\)\)$' % (key, val), re.M) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) + for (name, ver) in deps: - regex = re.compile('^\s*module load %s\s*$' % os.path.join(name, ver), re.M) + if get_module_syntax() == 'Tcl': + regex = re.compile(r'^\s*module load %s\s*$' % os.path.join(name, ver), re.M) + elif get_module_syntax() == 'Lua': + regex = re.compile(r'^\s*load\("%s"\)$' % os.path.join(name, ver), re.M) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) + for (name, ver) in hiddendeps: - regex = re.compile('^\s*module load %s/.%s\s*$' % (name, ver), re.M) + if get_module_syntax() == 'Tcl': + regex = re.compile(r'^\s*module load %s/.%s\s*$' % (name, ver), re.M) + elif get_module_syntax() == 'Lua': + regex = re.compile(r'^\s*load\("%s/.%s"\)$' % (name, ver), re.M) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) def test_gen_dirs(self): @@ -545,6 +589,9 @@ def test_exclude_path_to_top_of_module_tree(self): impi_modfile_path = os.path.join('Compiler', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049') imkl_modfile_path = os.path.join('MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049', 'imkl', '11.1.2.144') + if get_module_syntax() == 'Lua': + impi_modfile_path += '.lua' + imkl_modfile_path += '.lua' # example: for imkl on top of iimpi toolchain with HierarchicalMNS, no module load statements should be included # not for the toolchain or any of the toolchain components, @@ -588,15 +635,6 @@ def test_patch_step(self): eb.extract_step() eb.patch_step() - def tearDown(self): - """ make sure to remove the temporary file """ - super(EasyBlockTest, self).tearDown() - - os.remove(self.eb_file) - if self.orig_tmp_logdir is not None: - os.environ['EASYBUILD_TMP_LOGDIR'] = self.orig_tmp_logdir - shutil.rmtree(self.test_tmp_logdir, True) - def suite(): """ return all the tests in this file """ diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index e82d768112..6454cf1087 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -42,13 +42,15 @@ import easybuild.tools.build_log import easybuild.framework.easyconfig as easyconfig from easybuild.framework.easyblock import EasyBlock +from easybuild.framework.easyconfig.constants import EXTERNAL_MODULE_MARKER from easybuild.framework.easyconfig.easyconfig import EasyConfig from easybuild.framework.easyconfig.easyconfig import create_paths -from easybuild.framework.easyconfig.easyconfig import fetch_parameter_from_easyconfig_file, get_easyblock_class +from easybuild.framework.easyconfig.easyconfig import get_easyblock_class from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak_one from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import module_classes +from easybuild.tools.configobj import ConfigObj from easybuild.tools.filetools import read_file, write_file from easybuild.tools.module_naming_scheme.toolchain import det_toolchain_compilers, det_toolchain_mpi from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version @@ -212,6 +214,9 @@ def test_dependency(self): self.assertErrorRegex(EasyBuildError, "Dependency foo of unsupported type", eb._parse_dependency, "foo") self.assertErrorRegex(EasyBuildError, "without name", eb._parse_dependency, ()) self.assertErrorRegex(EasyBuildError, "without version", eb._parse_dependency, {'name': 'test'}) + err_msg = "Incorrect external dependency specification" + self.assertErrorRegex(EasyBuildError, err_msg, eb._parse_dependency, (EXTERNAL_MODULE_MARKER,)) + self.assertErrorRegex(EasyBuildError, err_msg, eb._parse_dependency, ('foo', '1.2.3', EXTERNAL_MODULE_MARKER)) def test_extra_options(self): """ extra_options should allow other variables to be stored """ @@ -367,7 +372,6 @@ def test_tweaking(self): 'toolchain_name': tcname, 'patches': new_patches[:1], 'homepage': homepage, - 'foo': "bar" } tweak_one(self.eb_file, tweaked_fn, tweaks) @@ -515,7 +519,7 @@ def test_obtain_easyconfig(self): 'toolchain_name': tcname, 'toolchain_version': tcver, 'version': ver, - 'foo': 'bar123' + 'start_dir': 'bar123' }) res = obtain_ec_for(specs, [self.ec_dir], None) self.assertEqual(res[1], "%s-%s-%s-%s%s.eb" % (name, ver, tcname, tcver, suff)) @@ -526,11 +530,16 @@ def test_obtain_easyconfig(self): self.assertEqual(ec['version'], specs['version']) self.assertEqual(ec['versionsuffix'], specs['versionsuffix']) self.assertEqual(ec['toolchain'], {'name': tcname, 'version': tcver}) - # can't check for key 'foo', because EasyConfig ignores parameter names it doesn't know about - txt = read_file(res[1]) - self.assertTrue(re.search('foo = "%s"' % specs['foo'], txt)) + self.assertEqual(ec['start_dir'], specs['start_dir']) os.remove(res[1]) + specs.update({ + 'foo': 'bar123' + }) + self.assertErrorRegex(EasyBuildError, "Unkown easyconfig parameter: foo", + obtain_ec_for, specs, [self.ec_dir], None) + del specs['foo'] + # should pick correct version, i.e. not newer than what's specified, if a choice needs to be made ver = '3.14' specs.update({'version': ver}) @@ -574,6 +583,8 @@ def test_obtain_easyconfig(self): 'short_mod_name': 'foo/1.2.3-GCC-4.4.5', 'full_mod_name': 'foo/1.2.3-GCC-4.4.5', 'hidden': False, + 'external_module': False, + 'external_module_metadata': {}, }, { 'name': 'bar', @@ -584,6 +595,8 @@ def test_obtain_easyconfig(self): 'short_mod_name': 'bar/666-gompi-1.4.10-bleh', 'full_mod_name': 'bar/666-gompi-1.4.10-bleh', 'hidden': False, + 'external_module': False, + 'external_module_metadata': {}, }, { 'name': 'test', @@ -594,6 +607,8 @@ def test_obtain_easyconfig(self): 'short_mod_name': 'test/.3.2.1-GCC-4.4.5', 'full_mod_name': 'test/.3.2.1-GCC-4.4.5', 'hidden': True, + 'external_module': False, + 'external_module_metadata': {}, }, ] @@ -874,11 +889,6 @@ def test_fetch_parameters_from_easyconfig(self): self.assertEqual(fetch_parameters_from_easyconfig(read_file(toy_ec_file), ['description'])[0], "Toy C program.") - # also check deprecated function fetch_parameter_from_easyconfig_file - os.environ['EASYBUILD_DEPRECATED'] = '2.0' - init_config() - self.assertEqual(fetch_parameter_from_easyconfig_file(toy_ec_file, 'description'), "Toy C program.") - def test_get_easyblock_class(self): """Test get_easyblock_class function.""" from easybuild.easyblocks.generic.configuremake import ConfigureMake @@ -1035,6 +1045,92 @@ def set_ec_key(key): ec[key] = 'foobar' self.assertErrorRegex(EasyBuildError, error_regex, set_ec_key, 'therenosucheasyconfigparameterlikethis') + def test_external_dependencies(self): + """Test specifying external (build) dependencies.""" + ectxt = read_file(os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0-deps.eb')) + toy_ec = os.path.join(self.test_prefix, 'toy-0.0-external-deps.eb') + + # just specify some of the test modules we ship, doesn't matter where they come from + ectxt += "\ndependencies += [('foobar/1.2.3', EXTERNAL_MODULE)]" + ectxt += "\nbuilddependencies = [('somebuilddep/0.1', EXTERNAL_MODULE)]" + write_file(toy_ec, ectxt) + + build_options = { + 'valid_module_classes': module_classes(), + 'external_modules_metadata': ConfigObj(), + } + init_config(build_options=build_options) + ec = EasyConfig(toy_ec) + + builddeps = ec.builddependencies() + self.assertEqual(len(builddeps), 1) + self.assertEqual(builddeps[0]['short_mod_name'], 'somebuilddep/0.1') + self.assertEqual(builddeps[0]['full_mod_name'], 'somebuilddep/0.1') + self.assertEqual(builddeps[0]['external_module'], True) + + deps = ec.dependencies() + self.assertEqual(len(deps), 4) + correct_deps = ['ictce/4.1.13', 'GCC/4.7.2', 'foobar/1.2.3', 'somebuilddep/0.1'] + self.assertEqual([d['short_mod_name'] for d in deps], correct_deps) + self.assertEqual([d['full_mod_name'] for d in deps], correct_deps) + self.assertEqual([d['external_module'] for d in deps], [False, True, True, True]) + + metadata = os.path.join(self.test_prefix, 'external_modules_metadata.cfg') + metadatatxt = '\n'.join(['[foobar/1.2.3]', 'name = foo,bar', 'version = 1.2.3,3.2.1', 'prefix = /foo/bar']) + write_file(metadata, metadatatxt) + cfg = init_config(args=['--external-modules-metadata=%s' % metadata]) + build_options = { + 'external_modules_metadata': cfg.external_modules_metadata, + 'valid_module_classes': module_classes(), + } + init_config(build_options=build_options) + ec = EasyConfig(toy_ec) + self.assertEqual(ec.dependencies()[2]['short_mod_name'], 'foobar/1.2.3') + self.assertEqual(ec.dependencies()[2]['external_module'], True) + metadata = { + 'name': ['foo', 'bar'], + 'version': ['1.2.3', '3.2.1'], + 'prefix': '/foo/bar', + } + self.assertEqual(ec.dependencies()[2]['external_module_metadata'], metadata) + + def test_update(self): + """Test use of update() method for EasyConfig instances.""" + toy_ebfile = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'toy-0.0.eb') + ec = EasyConfig(toy_ebfile) + + # for string values: append + ec.update('unpack_options', '--strip-components=1') + self.assertEqual(ec['unpack_options'].strip(), '--strip-components=1') + + ec.update('description', "- just a test") + self.assertEqual(ec['description'].strip(), "Toy C program. - just a test") + + # spaces in between multiple updates for stirng values + ec.update('configopts', 'CC="$CC"') + ec.update('configopts', 'CXX="$CXX"') + self.assertTrue(ec['configopts'].strip().endswith('CC="$CC" CXX="$CXX"')) + + # for list values: extend + ec.update('patches', ['foo.patch', 'bar.patch']) + self.assertEqual(ec['patches'], ['toy-0.0_typo.patch', 'foo.patch', 'bar.patch']) + + def test_hide_hidden_deps(self): + """Test use of --hide-deps on hiddendependencies.""" + test_dir = os.path.dirname(os.path.abspath(__file__)) + ec_file = os.path.join(test_dir, 'easyconfigs', 'gzip-1.4-GCC-4.6.3.eb') + ec = EasyConfig(ec_file) + self.assertEqual(ec['hiddendependencies'][0]['full_mod_name'], 'toy/.0.0-deps') + self.assertEqual(ec['dependencies'], []) + + build_options = { + 'hide_deps': ['toy'], + 'valid_module_classes': module_classes(), + } + init_config(build_options=build_options) + ec = EasyConfig(ec_file) + self.assertEqual(ec['hiddendependencies'][0]['full_mod_name'], 'toy/.0.0-deps') + self.assertEqual(ec['dependencies'], []) def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/easyconfigformat.py b/test/framework/easyconfigformat.py index 393cd3c4f4..e52175461e 100644 --- a/test/framework/easyconfigformat.py +++ b/test/framework/easyconfigformat.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyconfigparser.py b/test/framework/easyconfigparser.py index 5d0d4c5959..e163c9f0e0 100644 --- a/test/framework/easyconfigparser.py +++ b/test/framework/easyconfigparser.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyconfigs/toy-0.0-deps.eb b/test/framework/easyconfigs/toy-0.0-deps.eb index 646f9db067..84e6e22bc1 100644 --- a/test/framework/easyconfigs/toy-0.0-deps.eb +++ b/test/framework/easyconfigs/toy-0.0-deps.eb @@ -18,7 +18,10 @@ checksums = [[ ]] patches = ['toy-0.0_typo.patch'] -dependencies = [('ictce', '4.1.13', '', True)] +dependencies = [ + ('ictce', '4.1.13', '', True), + ('GCC/4.7.2', EXTERNAL_MODULE), +] sanity_check_paths = { 'files': [('bin/yot', 'bin/toy')], diff --git a/test/framework/easyconfigversion.py b/test/framework/easyconfigversion.py index 21411c7be2..a7331b4d45 100644 --- a/test/framework/easyconfigversion.py +++ b/test/framework/easyconfigversion.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2014 Ghent University +# Copyright 2014-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/ebconfigobj.py b/test/framework/ebconfigobj.py index 667d1ba7b0..cad6b30e73 100644 --- a/test/framework/ebconfigobj.py +++ b/test/framework/ebconfigobj.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2014 Ghent University +# Copyright 2014-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 1ad8fdcfe1..a6545454f6 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,7 +34,7 @@ import stat import tempfile import urllib2 -from test.framework.utilities import EnhancedTestCase +from test.framework.utilities import EnhancedTestCase, init_config from unittest import TestLoader, main import easybuild.tools.filetools as ft @@ -201,6 +201,18 @@ def test_download_file(self): res = ft.download_file(fn, source_url, target_location) self.assertEqual(res, target_location, "'download' of local file works after removing broken proxy") + # make sure specified timeout is parsed correctly (as a float, not a string) + opts = init_config(args=['--download-timeout=5.3']) + init_config(build_options={'download_timeout': opts.download_timeout}) + target_location = os.path.join(self.test_prefix, 'jenkins_robots.txt') + url = 'https://jenkins1.ugent.be/robots.txt' + try: + urllib2.urlopen(url) + res = ft.download_file(fn, url, target_location) + self.assertEqual(res, target_location, "download with specified timeout works") + except urllib2.URLError: + print "Skipping timeout test in test_download_file (working offline)" + def test_mkdir(self): """Test mkdir function.""" tmpdir = tempfile.mkdtemp() @@ -308,6 +320,36 @@ def test_guess_patch_level(self): ]: self.assertEqual(ft.guess_patch_level([patched_file], self.test_buildpath), correct_patch_level) + def test_move_logs(self): + """Test move_logs function.""" + fh, fp = tempfile.mkstemp() + os.close(fh) + ft.write_file(fp, 'foobar') + ft.write_file(fp + '.1', 'moarfoobar') + ft.move_logs(fp, os.path.join(self.test_prefix, 'foo.log')) + + self.assertEqual(ft.read_file(os.path.join(self.test_prefix, 'foo.log')), 'foobar') + self.assertEqual(ft.read_file(os.path.join(self.test_prefix, 'foo.log.1')), 'moarfoobar') + + ft.write_file(os.path.join(self.test_prefix, 'bar.log'), 'bar') + ft.write_file(os.path.join(self.test_prefix, 'bar.log_1'), 'barbar') + + fh, fp = tempfile.mkstemp() + os.close(fh) + ft.write_file(fp, 'moarbar') + ft.write_file(fp + '.1', 'evenmoarbar') + ft.move_logs(fp, os.path.join(self.test_prefix, 'bar.log')) + + logs = ['bar.log', 'bar.log.1', 'bar.log_0', 'bar.log_1', + os.path.basename(self.logfile), + 'foo.log', 'foo.log.1'] + self.assertEqual(sorted([f for f in os.listdir(self.test_prefix) if not f.startswith('tmp')]), logs) + self.assertEqual(ft.read_file(os.path.join(self.test_prefix, 'bar.log_0')), 'bar') + self.assertEqual(ft.read_file(os.path.join(self.test_prefix, 'bar.log_1')), 'barbar') + self.assertEqual(ft.read_file(os.path.join(self.test_prefix, 'bar.log')), 'moarbar') + self.assertEqual(ft.read_file(os.path.join(self.test_prefix, 'bar.log.1')), 'evenmoarbar') + + def suite(): """ returns all the testcases in this module """ return TestLoader().loadTestsFromTestCase(FileToolsTest) diff --git a/test/framework/format_convert.py b/test/framework/format_convert.py index 84c5c0f067..a9ae19f51e 100644 --- a/test/framework/format_convert.py +++ b/test/framework/format_convert.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2014 Ghent University +# Copyright 2014-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/general.py b/test/framework/general.py index d50b854d4d..545e37c062 100644 --- a/test/framework/general.py +++ b/test/framework/general.py @@ -28,12 +28,14 @@ @author: Kenneth hoste (Ghent University) """ import os +import re from test.framework.utilities import EnhancedTestCase from unittest import TestLoader, main import vsc import easybuild.framework +from easybuild.tools.filetools import read_file class GeneralTest(EnhancedTestCase): @@ -50,6 +52,23 @@ def test_vsc_location(self): msg = "vsc-base is not provided by EasyBuild framework itself, found location: %s" % vsc_loc self.assertFalse(os.path.samefile(framework_loc, vsc_loc), msg) + def test_error_reporting(self): + """Make sure error reporting is done correctly (no more log.error, log.exception).""" + # easybuild.framework.__file__ provides location to /easybuild/framework/__init__.py + easybuild_loc = os.path.dirname(os.path.dirname(os.path.abspath(easybuild.framework.__file__))) + + log_method_regexes = [ + re.compile("log\.error\("), + re.compile("log\.exception\("), + re.compile("log\.raiseException\("), + ] + + for dirpath, _, filenames in os.walk(easybuild_loc): + for filename in [f for f in filenames if f.endswith('.py')]: + path = os.path.join(dirpath, filename) + txt = read_file(path) + for regex in log_method_regexes: + self.assertFalse(regex.search(txt), "No match for '%s' in %s" % (regex.pattern, path)) def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/github.py b/test/framework/github.py index fb6cdbecb2..819b1d9a6e 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/license.py b/test/framework/license.py index 55c389a3db..026b0a722a 100644 --- a/test/framework/license.py +++ b/test/framework/license.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 4f8c3c7c74..5f6029cda5 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,14 +34,14 @@ import sys import tempfile from test.framework.utilities import EnhancedTestCase, init_config -from unittest import TestLoader, main +from unittest import TestLoader, TestSuite, TextTestRunner, main from vsc.utils.fancylogger import setLogLevelDebug, logToScreen from vsc.utils.missing import get_subclasses import easybuild.tools.module_generator from easybuild.framework.easyconfig.tools import process_easyconfig from easybuild.tools import config -from easybuild.tools.module_generator import ModuleGenerator +from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl from easybuild.tools.module_naming_scheme.utilities import is_valid_module_name from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig.easyconfig import EasyConfig, ActiveMNS @@ -50,10 +50,12 @@ class ModuleGeneratorTest(EnhancedTestCase): - """ testcase for ModuleGenerator """ + """Tests for module_generator module.""" + + MODULE_GENERATOR_CLASS = None def setUp(self): - """ initialize ModuleGenerator with test Application """ + """Test setup.""" super(ModuleGeneratorTest, self).setUp() # find .eb file eb_path = os.path.join(os.path.join(os.path.dirname(__file__), 'easyconfigs'), 'gzip-1.4.eb') @@ -62,84 +64,142 @@ def setUp(self): ec = EasyConfig(eb_full_path) self.eb = EasyBlock(ec) - self.modgen = ModuleGenerator(self.eb) + self.modgen = self.MODULE_GENERATOR_CLASS(self.eb) self.modgen.app.installdir = tempfile.mkdtemp(prefix='easybuild-modgen-test-') self.orig_module_naming_scheme = config.get_module_naming_scheme() - def tearDown(self): - """cleanup""" - super(ModuleGeneratorTest, self).tearDown() - os.remove(self.eb.logfile) - shutil.rmtree(self.modgen.app.installdir) - def test_descr(self): """Test generation of module description (which includes '#%Module' header).""" + gzip_txt = "gzip (GNU zip) is a popular data compression program as a replacement for compress " gzip_txt += "- Homepage: http://www.gzip.org/" - expected = '\n'.join([ - "#%Module", - "", - "proc ModulesHelp { } {", - " puts stderr { %s" % gzip_txt, - " }", - "}", - "", - "module-whatis {Description: %s}" % gzip_txt, - "", - "set root %s" % self.modgen.app.installdir, - "", - "conflict gzip", - "", - ]) + + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + expected = '\n'.join([ + "#%Module", + "proc ModulesHelp { } {", + " puts stderr { %s" % gzip_txt, + " }", + "}", + '', + "module-whatis {Description: %s}" % gzip_txt, + '', + "set root %s" % self.modgen.app.installdir, + '', + "conflict gzip", + '', + ]) + + else: + expected = '\n'.join([ + 'help([[%s]])' % gzip_txt, + "whatis([[Name: gzip]])" , + "whatis([[Version: 1.4]])" , + "whatis([[Description: %s]])" % gzip_txt, + "whatis([[Homepage: http://www.gzip.org/]])", + '', + 'local root = "%s"' % self.modgen.app.installdir, + '', + 'conflict("gzip")', + '', + ]) desc = self.modgen.get_description() self.assertEqual(desc, expected) def test_load(self): """Test load part in generated module file.""" - expected = [ - "", - "if { ![is-loaded mod_name] } {", - " module load mod_name", - "}", - "", - ] - self.assertEqual('\n'.join(expected), self.modgen.load_module("mod_name")) - - # with recursive unloading: no if is-loaded guard - init_config(build_options={'recursive_mod_unload': True}) - expected = [ - "", - "module load mod_name", - "", - ] - self.assertEqual('\n'.join(expected), self.modgen.load_module("mod_name")) + + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + expected = [ + '', + "if { ![ is-loaded mod_name ] } {", + " module load mod_name", + "}", + '', + ] + self.assertEqual('\n'.join(expected), self.modgen.load_module("mod_name")) + + # with recursive unloading: no if is-loaded guard + init_config(build_options={'recursive_mod_unload': True}) + expected = [ + '', + "module load mod_name", + '', + ] + self.assertEqual('\n'.join(expected), self.modgen.load_module("mod_name")) + else: + expected = '\n'.join([ + '', + 'if not isloaded("mod_name") then', + ' load("mod_name")', + 'end', + '', + ]) + self.assertEqual(expected,self.modgen.load_module("mod_name")) + + init_config(build_options={'recursive_mod_unload': True}) + expected = '\n'.join([ + '', + 'load("mod_name")', + '', + ]) + self.assertEqual(expected,self.modgen.load_module("mod_name")) def test_unload(self): """Test unload part in generated module file.""" - expected = '\n'.join([ - "", - "if { [is-loaded mod_name] } {", - " module unload mod_name", - "}", - "", - ]) - self.assertEqual(expected, self.modgen.unload_module("mod_name")) + + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + expected = '\n'.join([ + '', + "if { [ is-loaded mod_name ] } {", + " module unload mod_name", + "}", + '', + ]) + self.assertEqual(expected, self.modgen.unload_module("mod_name")) + else: + expected = '\n'.join([ + '', + 'if isloaded("mod_name") then', + ' unload("mod_name")', + "end", + '', + ]) + self.assertEqual(expected, self.modgen.unload_module("mod_name")) def test_prepend_paths(self): """Test generating prepend-paths statements.""" # test prepend_paths - expected = ''.join([ - "prepend-path\tkey\t\t$root/path1\n", - "prepend-path\tkey\t\t$root/path2\n", - ]) - self.assertEqual(expected, self.modgen.prepend_paths("key", ["path1", "path2"])) - expected = "prepend-path\tbar\t\t$root/foo\n" - self.assertEqual(expected, self.modgen.prepend_paths("bar", "foo")) + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + expected = ''.join([ + "prepend-path\tkey\t\t$root/path1\n", + "prepend-path\tkey\t\t$root/path2\n", + "prepend-path\tkey\t\t$root\n", + ]) + self.assertEqual(expected, self.modgen.prepend_paths("key", ["path1", "path2", ''])) + + expected = "prepend-path\tbar\t\t$root/foo\n" + self.assertEqual(expected, self.modgen.prepend_paths("bar", "foo")) - self.assertEqual("prepend-path\tkey\t\t/abs/path\n", self.modgen.prepend_paths("key", ["/abs/path"], allow_abs=True)) + res = self.modgen.prepend_paths("key", ["/abs/path"], allow_abs=True) + self.assertEqual("prepend-path\tkey\t\t/abs/path\n", res) + + else: + expected = ''.join([ + 'prepend_path("key", pathJoin(root, "path1"))\n', + 'prepend_path("key", pathJoin(root, "path2"))\n', + 'prepend_path("key", root)\n', + ]) + self.assertEqual(expected, self.modgen.prepend_paths("key", ["path1", "path2", ''])) + + expected = 'prepend_path("bar", pathJoin(root, "foo"))\n' + self.assertEqual(expected, self.modgen.prepend_paths("bar", "foo")) + + expected = 'prepend_path("key", "/abs/path")\n' + self.assertEqual(expected, self.modgen.prepend_paths("key", ["/abs/path"], allow_abs=True)) self.assertErrorRegex(EasyBuildError, "Absolute path %s/foo passed to prepend_paths " \ "which only expects relative paths." % self.modgen.app.installdir, @@ -147,44 +207,56 @@ def test_prepend_paths(self): def test_use(self): """Test generating module use statements.""" - expected = '\n'.join([ - "module use /some/path", - "module use /foo/bar/baz", - ]) - self.assertEqual(self.modgen.use(["/some/path", "/foo/bar/baz"]), expected) + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + expected = ''.join([ + "module use /some/path\n", + "module use /foo/bar/baz\n", + ]) + self.assertEqual(self.modgen.use(["/some/path", "/foo/bar/baz"]), expected) + else: + expected = ''.join([ + 'prepend_path("MODULEPATH", "/some/path")\n', + 'prepend_path("MODULEPATH", "/foo/bar/baz")\n', + ]) + self.assertEqual(self.modgen.use(["/some/path", "/foo/bar/baz"]), expected) + def test_env(self): """Test setting of environment variables.""" # test set_environment - self.assertEqual('setenv\tkey\t\t"value"\n', self.modgen.set_environment("key", "value")) - self.assertEqual("setenv\tkey\t\t'va\"lue'\n", self.modgen.set_environment("key", 'va"lue')) - self.assertEqual('setenv\tkey\t\t"va\'lue"\n', self.modgen.set_environment("key", "va'lue")) - self.assertEqual('setenv\tkey\t\t"""va"l\'ue"""\n', self.modgen.set_environment("key", """va"l'ue""")) - + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + self.assertEqual('setenv\tkey\t\t"value"\n', self.modgen.set_environment("key", "value")) + self.assertEqual("setenv\tkey\t\t'va\"lue'\n", self.modgen.set_environment("key", 'va"lue')) + self.assertEqual('setenv\tkey\t\t"va\'lue"\n', self.modgen.set_environment("key", "va'lue")) + self.assertEqual('setenv\tkey\t\t"""va"l\'ue"""\n', self.modgen.set_environment("key", """va"l'ue""")) + else: + self.assertEqual('setenv("key", "value")\n', self.modgen.set_environment("key", "value")) + def test_alias(self): """Test setting of alias in modulefiles.""" - # test set_alias - self.assertEqual('set-alias\tkey\t\t"value"\n', self.modgen.set_alias("key", "value")) - self.assertEqual("set-alias\tkey\t\t'va\"lue'\n", self.modgen.set_alias("key", 'va"lue')) - self.assertEqual('set-alias\tkey\t\t"va\'lue"\n', self.modgen.set_alias("key", "va'lue")) - self.assertEqual('set-alias\tkey\t\t"""va"l\'ue"""\n', self.modgen.set_alias("key", """va"l'ue""")) + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + # test set_alias + self.assertEqual('set-alias\tkey\t\t"value"\n', self.modgen.set_alias("key", "value")) + self.assertEqual("set-alias\tkey\t\t'va\"lue'\n", self.modgen.set_alias("key", 'va"lue')) + self.assertEqual('set-alias\tkey\t\t"va\'lue"\n', self.modgen.set_alias("key", "va'lue")) + self.assertEqual('set-alias\tkey\t\t"""va"l\'ue"""\n', self.modgen.set_alias("key", """va"l'ue""")) + else: + self.assertEqual('setalias("key", "value")\n', self.modgen.set_alias("key", "value")) def test_load_msg(self): """Test including a load message in the module file.""" - tcl_load_msg = '\n'.join([ - '', - "if [ module-info mode load ] {", - " puts stderr \"test \\$test \\$test", - "test \\$foo \\$bar\"", - "}", - '', - ]) - self.assertEqual(tcl_load_msg, self.modgen.msg_on_load('test $test \\$test\ntest $foo \\$bar')) - - def test_tcl_footer(self): - """Test including a Tcl footer.""" - tcltxt = 'puts stderr "foo"' - self.assertEqual(tcltxt, self.modgen.add_tcl_footer(tcltxt)) + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + tcl_load_msg = '\n'.join([ + '', + "if { [ module-info mode load ] } {", + " puts stderr \"test \\$test \\$test", + "test \\$foo \\$bar\"", + "}", + '', + ]) + self.assertEqual(tcl_load_msg, self.modgen.msg_on_load('test $test \\$test\ntest $foo \\$bar')) + else: + pass def test_module_naming_scheme(self): """Test using default module naming scheme.""" @@ -197,6 +269,7 @@ def test_module_naming_scheme(self): build_options = { 'check_osdeps': False, + 'external_modules_metadata': {}, 'robot_path': [ecs_dir], 'valid_stops': all_stops, 'validate': False, @@ -467,13 +540,25 @@ def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, init_modpaths): for ecfile, mns_vals in test_ecs.items(): test_ec(ecfile, *mns_vals) +class TclModuleGeneratorTest(ModuleGeneratorTest): + """Test for module_generator module for Tcl syntax.""" + MODULE_GENERATOR_CLASS = ModuleGeneratorTcl + + +class LuaModuleGeneratorTest(ModuleGeneratorTest): + """Test for module_generator module for Tcl syntax.""" + MODULE_GENERATOR_CLASS = ModuleGeneratorLua + def suite(): """ returns all the testcases in this module """ - return TestLoader().loadTestsFromTestCase(ModuleGeneratorTest) + suite = TestSuite() + suite.addTests(TestLoader().loadTestsFromTestCase(TclModuleGeneratorTest)) + suite.addTests(TestLoader().loadTestsFromTestCase(LuaModuleGeneratorTest)) + return suite if __name__ == '__main__': #logToScreen(enable=True) #setLogLevelDebug() - main() + TextTestRunner().run(suite()) diff --git a/test/framework/modules.py b/test/framework/modules.py index e098f30a42..487cbb8ccb 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -38,7 +38,10 @@ from unittest import TestLoader, main from easybuild.framework.easyblock import EasyBlock +from easybuild.framework.easyconfig.easyconfig import EasyConfig +from easybuild.tools import config from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import read_file, write_file from easybuild.tools.modules import get_software_root, get_software_version, get_software_libdir, modules_tool diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index 333ef8fd1f..6130e33564 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2014 Ghent University +# Copyright 2014-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -29,6 +29,7 @@ """ import os import re +import stat import tempfile from vsc.utils import fancylogger @@ -41,7 +42,7 @@ from easybuild.tools import config, modules from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option -from easybuild.tools.filetools import which +from easybuild.tools.filetools import which, write_file from easybuild.tools.modules import modules_tool, Lmod from test.framework.utilities import init_config @@ -95,7 +96,8 @@ def test_environment_command(self): # should never get here self.assertTrue(False, 'BrokenMockModulesTool should fail') except EasyBuildError, err: - self.assertTrue('command is not available' in str(err)) + err_msg = "command is not available" + self.assertTrue(err_msg in str(err), "'%s' found in: %s" % (err_msg, err)) os.environ[BrokenMockModulesTool.COMMAND_ENVIRONMENT] = MockModulesTool.COMMAND os.environ['module'] = "() { /bin/echo $*\n}" @@ -160,6 +162,9 @@ def test_lmod_specific(self): } init_config(build_options=build_options) + lmod = Lmod(testing=True) + self.assertEqual(lmod.cmd, lmod_abspath) + # drop any location where 'lmod' or 'spider' can be found from $PATH paths = os.environ.get('PATH', '').split(os.pathsep) new_paths = [] @@ -173,8 +178,18 @@ def test_lmod_specific(self): # make sure $MODULEPATH contains path that provides some modules os.environ['MODULEPATH'] = os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules')) - # initialize Lmod modules tool, pass full path to 'lmod' via $LMOD_CMD + # initialize Lmod modules tool, pass (fake) full path to 'lmod' via $LMOD_CMD + fake_path = os.path.join(self.test_installpath, 'lmod') + write_file(fake_path, '#!/bin/bash\necho "Modules based on Lua: Version %s " >&2' % Lmod.REQ_VERSION) + os.chmod(fake_path, stat.S_IRUSR|stat.S_IXUSR) + os.environ['LMOD_CMD'] = fake_path + init_config(build_options=build_options) + lmod = Lmod(testing=True) + self.assertEqual(lmod.cmd, fake_path) + + # use correct full path for 'lmod' via $LMOD_CMD os.environ['LMOD_CMD'] = lmod_abspath + init_config(build_options=build_options) lmod = Lmod(testing=True) # obtain list of availabe modules, should be non-empty diff --git a/test/framework/options.py b/test/framework/options.py index ae2273a1d1..cc2ba6ac0e 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -39,9 +39,11 @@ from urllib2 import URLError import easybuild.tools.build_log +import easybuild.tools.options from easybuild.framework.easyconfig import BUILD, CUSTOM, DEPENDENCIES, EXTENSIONS, FILEMANAGEMENT, LICENSE from easybuild.framework.easyconfig import MANDATORY, MODULES, OTHER, TOOLCHAIN from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import DEFAULT_MODULECLASSES, get_module_syntax from easybuild.tools.environment import modify_env from easybuild.tools.filetools import mkdir, read_file, write_file from easybuild.tools.github import fetch_github_token @@ -109,7 +111,7 @@ def test_no_args(self): outtxt = self.eb_main([]) - error_msg = "ERROR .* Please provide one or multiple easyconfig files," + error_msg = "ERROR Please provide one or multiple easyconfig files," error_msg += " or use software build options to make EasyBuild search for easyconfigs" self.assertTrue(re.search(error_msg, outtxt), "Error message when eb is run without arguments") @@ -126,9 +128,6 @@ def test_debug(self): res = re.search(' %s ' % log_msg_type, outtxt) self.assertTrue(res, "%s log messages are included when using %s: %s" % (log_msg_type, debug_arg, outtxt)) - modify_env(os.environ, self.orig_environ) - tempfile.tempdir = None - def test_info(self): """Test enabling info logging.""" @@ -147,12 +146,8 @@ def test_info(self): res = re.search(' %s ' % log_msg_type, outtxt) self.assertTrue(not res, "%s log messages are *not* included when using %s" % (log_msg_type, info_arg)) - modify_env(os.environ, self.orig_environ) - tempfile.tempdir = None - def test_quiet(self): """Test enabling quiet logging (errors only).""" - for quiet_arg in ['--quiet']: args = [ 'nosuchfile.eb', @@ -162,14 +157,13 @@ def test_quiet(self): for log_msg_type in ['ERROR']: res = re.search(' %s ' % log_msg_type, outtxt) - self.assertTrue(res, "%s log messages are included when using %s (outtxt: %s)" % (log_msg_type, quiet_arg, outtxt)) + msg = "%s log messages are included when using %s (outtxt: %s)" % (log_msg_type, quiet_arg, outtxt) + self.assertTrue(res, msg) for log_msg_type in ['DEBUG', 'INFO']: res = re.search(' %s ' % log_msg_type, outtxt) - self.assertTrue(not res, "%s log messages are *not* included when using %s (outtxt: %s)" % (log_msg_type, quiet_arg, outtxt)) - - modify_env(os.environ, self.orig_environ) - tempfile.tempdir = None + msg = "%s log messages are *not* included when using %s (outtxt: %s)" % (log_msg_type, quiet_arg, outtxt) + self.assertTrue(not res, msg) def test_force(self): """Test forcing installation even if the module is already available.""" @@ -189,10 +183,8 @@ def test_force(self): already_msg = "GCC/4.6.3 is already installed" self.assertTrue(re.search(already_msg, outtxt), "Already installed message without --force, outtxt: %s" % outtxt) - # clear log file, clean up environment + # clear log file write_file(self.logfile, '') - modify_env(os.environ, self.orig_environ) - tempfile.tempdir = None # check that --force works args = [ @@ -234,10 +226,8 @@ def test_skip(self): os.chdir(self.cwd) modules_tool().purge() # reinitialize modules tool with original $MODULEPATH, to avoid problems with future tests - modify_env(os.environ, self.orig_environ) os.environ['MODULEPATH'] = '' modules_tool() - tempfile.tempdir = None # check log message with --skip for non-existing module args = [ @@ -290,15 +280,13 @@ def check_args(job_args, passed_args=None): assertmsg = "Info log msg with job command template for --job (job_msg: %s, outtxt: %s)" % (job_msg, outtxt) self.assertTrue(re.search(job_msg, outtxt), assertmsg) - modify_env(os.environ, self.orig_environ) - tempfile.tempdir = None - # options passed are reordered, so order here matters to make tests pass check_args(['--debug']) check_args(['--debug', '--stop=configure', '--try-software-name=foo']) check_args(['--debug', '--robot-paths=/tmp/foo:/tmp/bar']) # --robot has preference over --robot-paths, --robot is not passed down - check_args(['--debug', '--robot-paths=/tmp/foo', '--robot=/tmp/bar'], passed_args=['--debug', '--robot-paths=/tmp/bar:/tmp/foo']) + check_args(['--debug', '--robot-paths=/tmp/foo', '--robot=/tmp/bar'], + passed_args=['--debug', '--robot-paths=/tmp/bar:/tmp/foo']) # 'zzz' prefix in the test name is intentional to make this test run last, # since it fiddles with the logging infrastructure which may break things @@ -335,8 +323,6 @@ def test_zzz_logtostdout(self): # cleanup os.remove(fn) - modify_env(os.environ, self.orig_environ) - tempfile.tempdir = None if os.path.exists(dummylogfn): os.remove(dummylogfn) @@ -370,6 +356,7 @@ def run_test(custom=None, extra_params=[], fmt=None): args.extend(['-e', custom]) outtxt = self.eb_main(args, logfile=dummylogfn, verbose=True) + logtxt = read_file(self.logfile) # check whether all parameter types are listed par_types = [BUILD, DEPENDENCIES, EXTENSIONS, FILEMANAGEMENT, @@ -380,21 +367,18 @@ def run_test(custom=None, extra_params=[], fmt=None): for param_type in [x[1] for x in par_types]: # regex for parameter group title, matches both txt and rst formats regex = re.compile("%s.*\n%s" % (param_type, '-' * len(param_type)), re.I) - tup = (param_type, avail_arg, args, outtxt) + tup = (param_type, avail_arg, args, logtxt) msg = "Parameter type %s is featured in output of eb %s (args: %s): %s" % tup - self.assertTrue(regex.search(outtxt), msg) + self.assertTrue(regex.search(logtxt), msg) # check a couple of easyconfig parameters for param in ["name", "version", "toolchain", "versionsuffix", "buildopts", "sources", "start_dir", "dependencies", "group", "exts_list", "moduleclass", "buildstats"] + extra_params: # regex for parameter name (with optional '*') & description, matches both txt and rst formats regex = re.compile("^[`]*%s(?:\*)?[`]*\s+\w+" % param, re.M) - tup = (param, avail_arg, args, regex.pattern, outtxt) + tup = (param, avail_arg, args, regex.pattern, logtxt) msg = "Parameter %s is listed with help in output of eb %s (args: %s, regex: %s): %s" % tup - self.assertTrue(regex.search(outtxt), msg) - - modify_env(os.environ, self.orig_environ) - tempfile.tempdir = None + self.assertTrue(regex.search(logtxt), msg) if os.path.exists(dummylogfn): os.remove(dummylogfn) @@ -420,7 +404,8 @@ def test__list_toolchains(self): outtxt = self.eb_main(args, logfile=dummylogfn) info_msg = r"INFO List of known toolchains \(toolchainname: module\[,module\.\.\.\]\):" - self.assertTrue(re.search(info_msg, outtxt), "Info message with list of known compiler toolchains") + logtxt = read_file(self.logfile) + self.assertTrue(re.search(info_msg, logtxt), "Info message with list of known compiler toolchains") # toolchain elements should be in alphabetical order tcs = { 'dummy': [], @@ -428,7 +413,7 @@ def test__list_toolchains(self): 'ictce': ['icc', 'ifort', 'imkl', 'impi'], } for tc, tcelems in tcs.items(): - res = re.findall("^\s*%s: .*" % tc, outtxt, re.M) + res = re.findall("^\s*%s: .*" % tc, logtxt, re.M) self.assertTrue(res, "Toolchain %s is included in list of known compiler toolchains" % tc) # every toolchain should only be mentioned once n = len(res) @@ -455,20 +440,18 @@ def test_avail_lists(self): '--unittest-file=%s' % self.logfile, ] outtxt = self.eb_main(args, logfile=dummylogfn) + logtxt = read_file(self.logfile) words = name.replace('-', ' ') info_msg = r"INFO List of supported %s:" % words - self.assertTrue(re.search(info_msg, outtxt), "Info message with list of available %s" % words) + self.assertTrue(re.search(info_msg, logtxt), "Info message with list of available %s" % words) for item in items: - res = re.findall("^\s*%s" % item, outtxt, re.M) + res = re.findall("^\s*%s" % item, logtxt, re.M) self.assertTrue(res, "%s is included in list of available %s" % (item, words)) # every item should only be mentioned once n = len(res) self.assertEqual(n, 1, "%s is only mentioned once (count: %d)" % (item, n)) - modify_env(os.environ, self.orig_environ) - tempfile.tempdir = None - if os.path.exists(dummylogfn): os.remove(dummylogfn) @@ -493,13 +476,14 @@ def test_avail_cfgfile_constants(self): '--unittest-file=%s' % self.logfile, ] outtxt = self.eb_main(args, logfile=dummylogfn) + logtxt = read_file(self.logfile) cfgfile_constants = { 'DEFAULT_ROBOT_PATHS': os.path.join(tmpdir, 'easybuild', 'easyconfigs'), } for cst_name, cst_value in cfgfile_constants.items(): - cst_regex = re.compile("^\*\s%s:\s.*\s\[value: .*%s.*\]" % (cst_name, cst_value), re.M) - tup = (cst_regex.pattern, outtxt) - self.assertTrue(cst_regex.search(outtxt), "Pattern '%s' in --avail-cfgfile_constants output: %s" % tup) + cst_regex = re.compile(r"^\*\s%s:\s.*\s\[value: .*%s.*\]" % (cst_name, cst_value), re.M) + tup = (cst_regex.pattern, logtxt) + self.assertTrue(cst_regex.search(logtxt), "Pattern '%s' in --avail-cfgfile_constants output: %s" % tup) if os.path.exists(dummylogfn): os.remove(dummylogfn) @@ -534,6 +518,7 @@ def test_list_easyblocks(self): '--unittest-file=%s' % self.logfile, ] outtxt = self.eb_main(args, logfile=dummylogfn) + logtxt = read_file(self.logfile) for pat in [ r"EasyBlock\n", @@ -541,10 +526,8 @@ def test_list_easyblocks(self): r"|--\s+bar\n", ]: - self.assertTrue(re.search(pat, outtxt), "Pattern '%s' is found in output of --list-easyblocks: %s" % (pat, outtxt)) - - modify_env(os.environ, self.orig_environ) - tempfile.tempdir = None + msg = "Pattern '%s' is found in output of --list-easyblocks: %s" % (pat, logtxt) + self.assertTrue(re.search(pat, logtxt), msg) # clear log write_file(self.logfile, '') @@ -555,6 +538,7 @@ def test_list_easyblocks(self): '--unittest-file=%s' % self.logfile, ] outtxt = self.eb_main(args, logfile=dummylogfn) + logtxt = read_file(self.logfile) for pat in [ r"EasyBlock\s+\(easybuild.framework.easyblock\)\n", @@ -562,7 +546,8 @@ def test_list_easyblocks(self): r"|--\s+bar\s+\(easybuild.easyblocks.generic.bar\)\n", ]: - self.assertTrue(re.search(pat, outtxt), "Pattern '%s' is found in output of --list-easyblocks: %s" % (pat, outtxt)) + msg = "Pattern '%s' is found in output of --list-easyblocks: %s" % (pat, logtxt) + self.assertTrue(re.search(pat, logtxt), msg) if os.path.exists(dummylogfn): os.remove(dummylogfn) @@ -578,16 +563,37 @@ def test_search(self): '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), '--unittest-file=%s' % self.logfile, ] - outtxt = self.eb_main(args, logfile=dummylogfn) + self.eb_main(args, logfile=dummylogfn) + logtxt = read_file(self.logfile) info_msg = r"Searching \(case-insensitive\) for 'gzip' in" - self.assertTrue(re.search(info_msg, outtxt), "Info message when searching for easyconfigs in '%s'" % outtxt) + self.assertTrue(re.search(info_msg, logtxt), "Info message when searching for easyconfigs in '%s'" % logtxt) for ec in ["gzip-1.4.eb", "gzip-1.4-GCC-4.6.3.eb"]: - self.assertTrue(re.search(" \* \S*%s$" % ec, outtxt, re.M), "Found easyconfig %s in '%s'" % (ec, outtxt)) + self.assertTrue(re.search(r" \* \S*%s$" % ec, logtxt, re.M), "Found easyconfig %s in '%s'" % (ec, logtxt)) + + if os.path.exists(dummylogfn): + os.remove(dummylogfn) + + write_file(self.logfile, '') + + args = [ + '--search=^gcc.*2.eb', + '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + '--unittest-file=%s' % self.logfile, + ] + self.eb_main(args, logfile=dummylogfn) + logtxt = read_file(self.logfile) + + info_msg = r"Searching \(case-insensitive\) for '\^gcc.\*2.eb' in" + self.assertTrue(re.search(info_msg, logtxt), "Info message when searching for easyconfigs in '%s'" % logtxt) + for ec in ['GCC-4.7.2.eb', 'GCC-4.8.2.eb', 'GCC-4.9.2.eb']: + self.assertTrue(re.search(r" \* \S*%s$" % ec, logtxt, re.M), "Found easyconfig %s in '%s'" % (ec, logtxt)) if os.path.exists(dummylogfn): os.remove(dummylogfn) + write_file(self.logfile, '') + for search_arg in ['-S', '--search-short']: open(self.logfile, 'w').write('') args = [ @@ -597,13 +603,14 @@ def test_search(self): os.path.join(os.path.dirname(__file__), 'easyconfigs'), '--unittest-file=%s' % self.logfile, ] - outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True, verbose=True) + self.eb_main(args, logfile=dummylogfn, raise_error=True, verbose=True) + logtxt = read_file(self.logfile) info_msg = r"Searching \(case-insensitive\) for 'toy-0.0' in" - self.assertTrue(re.search(info_msg, outtxt), "Info message when searching for easyconfigs in '%s'" % outtxt) - self.assertTrue(re.search('INFO CFGS\d+=', outtxt), "CFGS line message found in '%s'" % outtxt) + self.assertTrue(re.search(info_msg, logtxt), "Info message when searching for easyconfigs in '%s'" % logtxt) + self.assertTrue(re.search('INFO CFGS\d+=', logtxt), "CFGS line message found in '%s'" % logtxt) for ec in ["toy-0.0.eb", "toy-0.0-multiple.eb"]: - self.assertTrue(re.search(" \* \$CFGS\d+/*%s" % ec, outtxt), "Found easyconfig %s in '%s'" % (ec, outtxt)) + self.assertTrue(re.search(" \* \$CFGS\d+/*%s" % ec, logtxt), "Found easyconfig %s in '%s'" % (ec, logtxt)) if os.path.exists(dummylogfn): os.remove(dummylogfn) @@ -619,17 +626,18 @@ def test_dry_run(self): '--unittest-file=%s' % self.logfile, '--robot-paths=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), ] - outtxt = self.eb_main(args, logfile=dummylogfn) + self.eb_main(args, logfile=dummylogfn) + logtxt = read_file(self.logfile) info_msg = r"Dry run: printing build status of easyconfigs and dependencies" - self.assertTrue(re.search(info_msg, outtxt, re.M), "Info message dry running in '%s'" % outtxt) + self.assertTrue(re.search(info_msg, logtxt, re.M), "Info message dry running in '%s'" % logtxt) ecs_mods = [ ("gzip-1.4-GCC-4.6.3.eb", "gzip/1.4-GCC-4.6.3", ' '), ("GCC-4.6.3.eb", "GCC/4.6.3", 'x'), ] for ec, mod, mark in ecs_mods: regex = re.compile(r" \* \[%s\] \S+%s \(module: %s\)" % (mark, ec, mod), re.M) - self.assertTrue(regex.search(outtxt), "Found match for pattern %s in '%s'" % (regex.pattern, outtxt)) + self.assertTrue(regex.search(logtxt), "Found match for pattern %s in '%s'" % (regex.pattern, logtxt)) def test_dry_run_short(self): """Test dry run (short format).""" @@ -848,7 +856,7 @@ def test_from_pr(self): regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M) self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules)) - pr_tmpdir = os.path.join(tmpdir, 'easybuild-\S{6}', 'files_pr1239') + pr_tmpdir = os.path.join(tmpdir, 'eb-\S{6}', 'files_pr1239') regex = re.compile("Prepended list of robot search paths with %s:" % pr_tmpdir, re.M) self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) except URLError, err: @@ -882,7 +890,7 @@ def test_from_pr_listed_ecs(self): '--from-pr=1239', '--dry-run', # an argument must be specified to --robot, since easybuild-easyconfigs may not be installed - '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + '--robot=%s' % test_ecs_path, '--unittest-file=%s' % self.logfile, '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user '--tmpdir=%s' % tmpdir, @@ -890,13 +898,13 @@ def test_from_pr_listed_ecs(self): try: outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) modules = [ - (ecstmpdir, 'toy/0.0'), - ('.*', 'GCC/4.9.2'), # not included in PR + (test_ecs_path, 'toy/0.0'), # not included in PR + (test_ecs_path, 'GCC/4.9.2'), # not included in PR (tmpdir, 'hwloc/1.10.0-GCC-4.9.2'), (tmpdir, 'numactl/2.0.10-GCC-4.9.2'), (tmpdir, 'OpenMPI/1.8.4-GCC-4.9.2'), (tmpdir, 'gompi/2015a'), - ('.*', 'GCC/4.6.3'), + (test_ecs_path, 'GCC/4.6.3'), # not included in PR ] for path_prefix, module in modules: ec_fn = "%s.eb" % '-'.join(module.split('/')) @@ -922,9 +930,10 @@ def test_no_such_software(self): outtxt = self.eb_main(args) # error message when template is not found - error_msg1 = "ERROR .* No easyconfig files found for software nosuchsoftware, and no templates available. I'm all out of ideas." + error_msg1 = "ERROR No easyconfig files found for software nosuchsoftware, and no templates available. " + error_msg1 += "I'm all out of ideas." # error message when template is found - error_msg2 = "ERROR .* Unable to find an easyconfig for the given specifications" + error_msg2 = "ERROR Unable to find an easyconfig for the given specifications" msg = "Error message when eb can't find software with specified name (outtxt: %s)" % outtxt self.assertTrue(re.search(error_msg1, outtxt) or re.search(error_msg2, outtxt), msg) @@ -932,15 +941,22 @@ def test_footer(self): """Test specifying a module footer.""" # create file containing modules footer - module_footer_txt = '\n'.join([ - "# test footer", - "setenv SITE_SPECIFIC_ENV_VAR foobar", - ]) + if get_module_syntax() == 'Tcl': + module_footer_txt = '\n'.join([ + "# test footer", + "setenv SITE_SPECIFIC_ENV_VAR foobar", + ]) + elif get_module_syntax() == 'Lua': + module_footer_txt = '\n'.join([ + "-- test footer", + 'setenv("SITE_SPECIFIC_ENV_VAR", "foobar")', + ]) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + fd, modules_footer = tempfile.mkstemp(prefix='modules-footer-') os.close(fd) - f = open(modules_footer, 'w') - f.write(module_footer_txt) - f.close() + write_file(modules_footer, module_footer_txt) # use toy-0.0.eb easyconfig file that comes with the tests eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb') @@ -958,8 +974,10 @@ def test_footer(self): self.eb_main(args, do_build=True) toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_module += '.lua' toy_module_txt = read_file(toy_module) - footer_regex = re.compile(r'%s$' % module_footer_txt, re.M) + footer_regex = re.compile(r'%s$' % module_footer_txt.replace('(', '\\(').replace(')', '\\)'), re.M) msg = "modules footer '%s' is present in '%s'" % (module_footer_txt, toy_module_txt) self.assertTrue(footer_regex.search(toy_module_txt), msg) @@ -985,6 +1003,8 @@ def test_recursive_module_unload(self): self.eb_main(args, do_build=True, verbose=True) toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-deps') + if get_module_syntax() == 'Lua': + toy_module += '.lua' toy_module_txt = read_file(toy_module) is_loaded_regex = re.compile(r"if { !\[is-loaded gompi/1.3.12\] }", re.M) self.assertFalse(is_loaded_regex.search(toy_module_txt), "Recursive unloading is used: %s" % toy_module_txt) @@ -1007,19 +1027,19 @@ def test_tmpdir(self): '--debug', '--tmpdir=%s' % tmpdir, ] - outtxt = self.eb_main(args, do_build=True) + outtxt = self.eb_main(args, do_build=True, reset_env=False) - tmpdir_msg = r"Using %s\S+ as temporary directory" % os.path.join(tmpdir, 'easybuild-') + tmpdir_msg = r"Using %s\S+ as temporary directory" % os.path.join(tmpdir, 'eb-') found = re.search(tmpdir_msg, outtxt, re.M) self.assertTrue(found, "Log message for tmpdir found in outtxt: %s" % outtxt) for var in ['TMPDIR', 'TEMP', 'TMP']: - self.assertTrue(os.environ[var].startswith(os.path.join(tmpdir, 'easybuild-'))) - self.assertTrue(tempfile.gettempdir().startswith(os.path.join(tmpdir, 'easybuild-'))) + self.assertTrue(os.environ[var].startswith(os.path.join(tmpdir, 'eb-'))) + self.assertTrue(tempfile.gettempdir().startswith(os.path.join(tmpdir, 'eb-'))) tempfile_tmpdir = tempfile.mkdtemp() - self.assertTrue(tempfile_tmpdir.startswith(os.path.join(tmpdir, 'easybuild-'))) + self.assertTrue(tempfile_tmpdir.startswith(os.path.join(tmpdir, 'eb-'))) fd, tempfile_tmpfile = tempfile.mkstemp() - self.assertTrue(tempfile_tmpfile.startswith(os.path.join(tmpdir, 'easybuild-'))) + self.assertTrue(tempfile_tmpfile.startswith(os.path.join(tmpdir, 'eb-'))) # cleanup os.close(fd) @@ -1169,6 +1189,7 @@ def test_allow_modules_tool_mismatch(self): args = [ ec_file, '--modules-tool=MockModulesTool', + '--module-syntax=Tcl', # Lua would require Lmod ] self.eb_main(args, do_build=True) outtxt = read_file(self.logfile) @@ -1180,6 +1201,7 @@ def test_allow_modules_tool_mismatch(self): args = [ ec_file, '--modules-tool=MockModulesTool', + '--module-syntax=Tcl', # Lua would require Lmod '--allow-modules-tool-mismatch', ] self.eb_main(args, do_build=True) @@ -1192,6 +1214,7 @@ def test_allow_modules_tool_mismatch(self): args = [ ec_file, '--modules-tool=MockModulesTool', + '--module-syntax=Tcl', # Lua would require Lmod '--debug', ] self.eb_main(args, do_build=True) @@ -1303,7 +1326,7 @@ def test_recursive_try(self): mod = ec_name.replace('-', '/') else: mod = '%s-gompi-1.4.10' % ec_name.replace('-', '/') - mod_regex = re.compile("^ \* \[ \] \S+/easybuild-\S+/%s \(module: .*%s\)$" % (ec, mod), re.M) + mod_regex = re.compile("^ \* \[ \] \S+/eb-\S+/%s \(module: .*%s\)$" % (ec, mod), re.M) #mod_regex = re.compile("%s \(module: .*%s\)$" % (ec, mod), re.M) self.assertTrue(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) @@ -1376,7 +1399,44 @@ def test_filter_deps(self): self.assertFalse(re.search('module: FFTW/3.3.3-gompi', outtxt)) self.assertFalse(re.search('module: ScaLAPACK/2.0.2-gompi', outtxt)) self.assertFalse(re.search('module: zlib', outtxt)) - + + def test_hide_deps(self): + """Test use of --hide-deps.""" + test_dir = os.path.dirname(os.path.abspath(__file__)) + ec_file = os.path.join(test_dir, 'easyconfigs', 'goolf-1.4.10.eb') + os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules') + args = [ + ec_file, + '--buildpath=%s' % self.test_buildpath, + '--installpath=%s' % self.test_installpath, + '--robot=%s' % os.path.join(test_dir, 'easyconfigs'), + '--dry-run', + ] + outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True) + self.assertTrue(re.search('module: GCC/4.7.2', outtxt)) + self.assertTrue(re.search('module: OpenMPI/1.6.4-GCC-4.7.2', outtxt)) + self.assertTrue(re.search('module: OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', outtxt)) + self.assertTrue(re.search('module: FFTW/3.3.3-gompi', outtxt)) + self.assertTrue(re.search('module: ScaLAPACK/2.0.2-gompi', outtxt)) + # zlib is not a dep at all + self.assertFalse(re.search('module: zlib', outtxt)) + + # clear log file + open(self.logfile, 'w').write('') + + # filter deps (including a non-existing dep, i.e. zlib) + args.append('--hide-deps=FFTW,ScaLAPACK,zlib') + outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True) + self.assertTrue(re.search('module: GCC/4.7.2', outtxt)) + self.assertTrue(re.search('module: OpenMPI/1.6.4-GCC-4.7.2', outtxt)) + self.assertTrue(re.search('module: OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', outtxt)) + self.assertFalse(re.search(r'module: FFTW/3\.3\.3-gompi', outtxt)) + self.assertTrue(re.search(r'module: FFTW/\.3\.3\.3-gompi', outtxt)) + self.assertFalse(re.search(r'module: ScaLAPACK/2\.0\.2-gompi', outtxt)) + self.assertTrue(re.search(r'module: ScaLAPACK/\.2\.0\.2-gompi', outtxt)) + # zlib is not a dep at all + self.assertFalse(re.search(r'module: zlib', outtxt)) + def test_test_report_env_filter(self): """Test use of --test-report-env-filter.""" @@ -1442,7 +1502,7 @@ def test_robot(self): eb_file, '--robot-paths=%s' % test_ecs_path, ] - error_regex ='no module .* found for dependency' + error_regex = 'no module .* found for dependency' self.assertErrorRegex(EasyBuildError, error_regex, self.eb_main, args, raise_error=True, do_build=True) # enable robot, but without passing path required to resolve toy dependency => FAIL @@ -1484,6 +1544,141 @@ def test_robot(self): ec_regex = re.compile(r'^\s\*\s\[[xF ]\]\s%s' % os.path.join(test_ecs_path, ecfile), re.M) self.assertTrue(ec_regex.search(outtxt), "Pattern %s found in %s" % (ec_regex.pattern, outtxt)) + def test_missing_cfgfile(self): + """Test behaviour when non-existing config file is specified.""" + args = ['--configfiles=/no/such/cfgfile.foo'] + error_regex = "parseconfigfiles: configfile .* not found" + self.assertErrorRegex(EasyBuildError, error_regex, self.eb_main, args, raise_error=True) + + def test_show_default_moduleclasses(self): + """Test --show-default-moduleclasses.""" + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + args = [ + '--unittest-file=%s' % self.logfile, + '--show-default-moduleclasses', + ] + write_file(self.logfile, '') + self.eb_main(args, logfile=dummylogfn, verbose=True) + logtxt = read_file(self.logfile) + + lst = ["\t%s:[ ]*%s" % (c, d.replace('(', '\\(').replace(')', '\\)')) for (c, d) in DEFAULT_MODULECLASSES] + regex = re.compile("Default available module classes:\n\n" + '\n'.join(lst), re.M) + + self.assertTrue(regex.search(logtxt), "Pattern '%s' found in %s" % (regex.pattern, logtxt)) + + def test_show_default_configfiles(self): + """Test --show-default-configfiles.""" + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + home = os.environ['HOME'] + for envvar in ['XDG_CONFIG_DIRS', 'XDG_CONFIG_HOME']: + if envvar in os.environ: + del os.environ[envvar] + reload(easybuild.tools.options) + + args = [ + '--unittest-file=%s' % self.logfile, + '--show-default-configfiles', + ] + + cfgtxt = '\n'.join([ + '[config]', + 'prefix = %s' % self.test_prefix, + ]) + + expected_tmpl = '\n'.join([ + "Default list of configuration files:", + '', + "[with $XDG_CONFIG_HOME: %s, $XDG_CONFIG_DIRS: %s]", + '', + "* user-level: ${XDG_CONFIG_HOME:-$HOME/.config}/easybuild/config.cfg", + " -> %s", + "* system-level: ${XDG_CONFIG_DIRS:-/etc}/easybuild.d/*.cfg", + " -> %s/easybuild.d/*.cfg => ", + ]) + + write_file(self.logfile, '') + self.eb_main(args, logfile=dummylogfn, verbose=True) + logtxt = read_file(self.logfile) + + homecfgfile = os.path.join(os.environ['HOME'], '.config', 'easybuild', 'config.cfg') + homecfgfile_str = homecfgfile + if os.path.exists(homecfgfile): + homecfgfile_str += " => found" + else: + homecfgfile_str += " => not found" + expected = expected_tmpl % ('(not set)', '(not set)', homecfgfile_str, '{/etc}') + self.assertTrue(expected in logtxt) + + # to predict the full output, we need to take control over $HOME and $XDG_CONFIG_DIRS + os.environ['HOME'] = self.test_prefix + xdg_config_dirs = os.path.join(self.test_prefix, 'etc') + os.environ['XDG_CONFIG_DIRS'] = xdg_config_dirs + + expected_tmpl += '\n'.join([ + "%s", + '', + "Default list of existing configuration files (%d): %s", + ]) + + # put dummy cfgfile in place in $HOME (to predict last line of output which only lists *existing* files) + mkdir(os.path.join(self.test_prefix, '.config', 'easybuild'), parents=True) + homecfgfile = os.path.join(self.test_prefix, '.config', 'easybuild', 'config.cfg') + write_file(homecfgfile, cfgtxt) + + reload(easybuild.tools.options) + write_file(self.logfile, '') + self.eb_main(args, logfile=dummylogfn, verbose=True) + logtxt = read_file(self.logfile) + expected = expected_tmpl % ('(not set)', xdg_config_dirs, "%s => found" % homecfgfile, '{%s}' % xdg_config_dirs, + '(no matches)', 1, homecfgfile) + self.assertTrue(expected in logtxt) + + xdg_config_home = os.path.join(self.test_prefix, 'home') + os.environ['XDG_CONFIG_HOME'] = xdg_config_home + xdg_config_dirs = [os.path.join(self.test_prefix, 'etc'), os.path.join(self.test_prefix, 'moaretc')] + os.environ['XDG_CONFIG_DIRS'] = os.pathsep.join(xdg_config_dirs) + + # put various dummy cfgfiles in place + cfgfiles = [ + os.path.join(self.test_prefix, 'etc', 'easybuild.d', 'config.cfg'), + os.path.join(self.test_prefix, 'moaretc', 'easybuild.d', 'bar.cfg'), + os.path.join(self.test_prefix, 'moaretc', 'easybuild.d', 'foo.cfg'), + os.path.join(xdg_config_home, 'easybuild', 'config.cfg'), + ] + for cfgfile in cfgfiles: + mkdir(os.path.dirname(cfgfile), parents=True) + write_file(cfgfile, cfgtxt) + reload(easybuild.tools.options) + + write_file(self.logfile, '') + self.eb_main(args, logfile=dummylogfn, verbose=True) + logtxt = read_file(self.logfile) + expected = expected_tmpl % (xdg_config_home, os.pathsep.join(xdg_config_dirs), + "%s => found" % os.path.join(xdg_config_home, 'easybuild', 'config.cfg'), + '{' + ', '.join(xdg_config_dirs) + '}', + ', '.join(cfgfiles[:-1]), 4, ', '.join(cfgfiles)) + self.assertTrue(expected in logtxt) + + del os.environ['XDG_CONFIG_DIRS'] + del os.environ['XDG_CONFIG_HOME'] + os.environ['HOME'] = home + reload(easybuild.tools.options) + + def test_generate_cmd_line(self): + """Test for generate_cmd_line.""" + ebopts = EasyBuildOptions() + self.assertEqual(ebopts.generate_cmd_line(), []) + + ebopts = EasyBuildOptions(go_args=['--force']) + self.assertEqual(ebopts.generate_cmd_line(), ['--force']) + + ebopts = EasyBuildOptions(go_args=['--search=bar', '--search', 'foobar']) + self.assertEqual(ebopts.generate_cmd_line(), ['--search=foobar']) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/repository.py b/test/framework/repository.py index 08ee4ac151..5622e49f65 100644 --- a/test/framework/repository.py +++ b/test/framework/repository.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -40,6 +40,7 @@ from easybuild.tools.repository.svnrepo import SvnRepository from easybuild.tools.repository.repository import init_repository from easybuild.tools.run import run_cmd +from easybuild.tools.version import VERSION class RepositoryTest(EnhancedTestCase): @@ -97,6 +98,11 @@ def test_gitrepo(self): repo.init() toy_ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb') repo.add_easyconfig(toy_ec_file, 'test', '1.0', {}, False) + repo.commit("toy/0.0") + + log_regex = re.compile(r"toy/0.0 with EasyBuild v%s @ .* \(time: .*, user: .*\)" % VERSION, re.M) + logmsg = repo.client.log('HEAD^!') + self.assertTrue(log_regex.search(logmsg), "Pattern '%s' found in %s" % (log_regex.pattern, logmsg)) shutil.rmtree(repo.wc) shutil.rmtree(tmpdir) diff --git a/test/framework/robot.py b/test/framework/robot.py index c157a25fad..5898160f9c 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -29,6 +29,9 @@ """ import os +import re +import shutil +import tempfile from copy import deepcopy from test.framework.utilities import EnhancedTestCase, init_config from unittest import TestLoader @@ -39,9 +42,15 @@ from easybuild.framework.easyconfig.tools import skip_available from easybuild.tools import config, modules from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import read_file, write_file +from easybuild.tools.github import fetch_github_token from easybuild.tools.robot import resolve_dependencies from test.framework.utilities import find_full_path + +# test account, for which a token is available +GITHUB_TEST_ACCOUNT = 'easybuild_test' + ORIG_MODULES_TOOL = modules.modules_tool ORIG_ECTOOLS_MODULES_TOOL = ectools.modules_tool ORIG_ROBOT_MODULES_TOOL = robot.modules_tool @@ -79,8 +88,12 @@ class RobotTest(EnhancedTestCase): """ Testcase for the robot dependency resolution """ def setUp(self): - """Set up everything for a unit test.""" + """Set up test.""" super(RobotTest, self).setUp() + self.github_token = fetch_github_token(GITHUB_TEST_ACCOUNT) + + def xtest_resolve_dependencies(self): + """ Test with some basic testcases (also check if he can find dependencies inside the given directory """ # replace Modules class with something we have control over config.modules_tool = mock_module @@ -88,11 +101,9 @@ def setUp(self): robot.modules_tool = mock_module os.environ['module'] = "() { eval `/bin/echo $*`\n}" - self.base_easyconfig_dir = find_full_path(os.path.join("test", "framework", "easyconfigs")) - self.assertTrue(self.base_easyconfig_dir) + base_easyconfig_dir = find_full_path(os.path.join("test", "framework", "easyconfigs")) + self.assertTrue(base_easyconfig_dir) - def test_resolve_dependencies(self): - """ Test with some basic testcases (also check if he can find dependencies inside the given directory """ easyconfig = { 'spec': '_', 'full_mod_name': 'name/version', @@ -128,7 +139,7 @@ def test_resolve_dependencies(self): }], 'parsed': True, } - build_options.update({'robot': True, 'robot_path': self.base_easyconfig_dir}) + build_options.update({'robot': True, 'robot_path': base_easyconfig_dir}) init_config(build_options=build_options) res = resolve_dependencies([deepcopy(easyconfig_dep)]) # dependency should be found, order should be correct @@ -188,7 +199,7 @@ def test_resolve_dependencies(self): 'hidden': False, }] ecs = [deepcopy(easyconfig_dep)] - build_options.update({'robot_path': self.base_easyconfig_dir}) + build_options.update({'robot_path': base_easyconfig_dir}) init_config(build_options=build_options) res = resolve_dependencies([deepcopy(easyconfig_dep)]) @@ -288,10 +299,6 @@ def test_resolve_dependencies(self): self.assertEqual('goolf/1.4.10', res[2]['full_mod_name']) self.assertEqual('foo/1.2.3', res[3]['full_mod_name']) - def tearDown(self): - """ reset the Modules back to its original """ - super(RobotTest, self).tearDown() - config.modules_tool = ORIG_MODULES_TOOL ectools.modules_tool = ORIG_ECTOOLS_MODULES_TOOL robot.modules_tool = ORIG_ROBOT_MODULES_TOOL @@ -301,6 +308,103 @@ def tearDown(self): if 'module' in os.environ: del os.environ['module'] + def test_det_easyconfig_paths(self): + """Test det_easyconfig_paths function (without --from-pr).""" + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + + test_ec = 'toy-0.0-deps.eb' + shutil.copy2(os.path.join(test_ecs_path, test_ec), self.test_prefix) + shutil.copy2(os.path.join(test_ecs_path, 'ictce-4.1.13.eb'), self.test_prefix) + self.assertFalse(os.path.exists(test_ec)) + + args = [ + os.path.join(test_ecs_path, 'toy-0.0.eb'), + test_ec, # relative path, should be resolved via robot search path + # PR for foss/2015a, see https://github.com/hpcugent/easybuild-easyconfigs/pull/1239/files + #'--from-pr=1239', + '--dry-run', + '--debug', + '--robot', + '--robot-paths=%s' % self.test_prefix, # override $EASYBUILD_ROBOT_PATHS + '--unittest-file=%s' % self.logfile, + '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user + '--tmpdir=%s' % self.test_prefix, + ] + outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) + + modules = [ + (test_ecs_path, 'toy/0.0'), # specified easyconfigs, available at given location + (self.test_prefix, 'ictce/4.1.13'), # dependency, found in robot search path + (self.test_prefix, 'toy/0.0-deps'), # specified easyconfig, found in robot search path + ] + for path_prefix, module in modules: + ec_fn = "%s.eb" % '-'.join(module.split('/')) + regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path_prefix, ec_fn, module), re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + + def test_det_easyconfig_paths_from_pr(self): + """Test det_easyconfig_paths function, with --from-pr enabled as well.""" + if self.github_token is None: + print "Skipping test_from_pr, no GitHub token available?" + return + + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + + test_ec = 'toy-0.0-deps.eb' + shutil.copy2(os.path.join(test_ecs_path, test_ec), self.test_prefix) + shutil.copy2(os.path.join(test_ecs_path, 'ictce-4.1.13.eb'), self.test_prefix) + self.assertFalse(os.path.exists(test_ec)) + + gompi_2015a_txt = '\n'.join([ + "easyblock = 'Toolchain'", + "name = 'gompi'", + "version = '2015a'", + "versionsuffix = '-test'", + "homepage = 'foo'", + "description = 'bar'", + "toolchain = {'name': 'dummy', 'version': 'dummy'}", + ]) + write_file(os.path.join(self.test_prefix, 'gompi-2015a-test.eb'), gompi_2015a_txt) + # put gompi-2015a.eb easyconfig in place that shouldn't be considered (paths via --from-pr have precedence) + write_file(os.path.join(self.test_prefix, 'gompi-2015a.eb'), gompi_2015a_txt) + + args = [ + os.path.join(test_ecs_path, 'toy-0.0.eb'), + test_ec, # relative path, should be resolved via robot search path + # PR for foss/2015a, see https://github.com/hpcugent/easybuild-easyconfigs/pull/1239/files + '--from-pr=1239', + 'FFTW-3.3.4-gompi-2015a.eb', + 'gompi-2015a-test.eb', # relative path, available in robot search path + '--dry-run', + '--robot', + '--robot=%s' % self.test_prefix, + '--unittest-file=%s' % self.logfile, + '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user + '--tmpdir=%s' % self.test_prefix, + ] + outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) + + from_pr_prefix = os.path.join(self.test_prefix, '.*', 'files_pr1239') + modules = [ + (test_ecs_path, 'toy/0.0'), # specified easyconfigs, available at given location + (self.test_prefix, 'ictce/4.1.13'), # dependency, found in robot search path + (self.test_prefix, 'toy/0.0-deps'), # specified easyconfig, found in robot search path + (self.test_prefix, 'gompi/2015a-test'), # specified easyconfig, found in robot search path + (from_pr_prefix, 'FFTW/3.3.4-gompi-2015a'), # part of PR easyconfigs + (from_pr_prefix, 'gompi/2015a'), # part of PR easyconfigs + (test_ecs_path, 'GCC/4.9.2'), # dependency for PR easyconfigs, found in robot search path + ] + for path_prefix, module in modules: + ec_fn = "%s.eb" % '-'.join(module.split('/')) + regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path_prefix, ec_fn, module), re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/run.py b/test/framework/run.py index ba8cb3c491..4d46392ddf 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -95,30 +95,6 @@ def test_parse_log_error(self): errors = parse_log_for_error("error failed", True) self.assertEqual(len(errors), 1) - def test_run_cmd_suse(self): - """Test run_cmd on SuSE systems, which have $PROFILEREAD set.""" - # avoid warning messages - run_log_level = run_log.getEffectiveLevel() - run_log.setLevel('ERROR') - - # run_cmd should also work if $PROFILEREAD is set (very relevant for SuSE systems) - profileread = os.environ.get('PROFILEREAD', None) - os.environ['PROFILEREAD'] = 'profilereadxxx' - try: - (out, ec) = run_cmd("echo hello") - except Exception, err: - out, ec = "ERROR: %s" % err, 1 - - # make sure it's restored again before we can fail the test - if profileread is not None: - os.environ['PROFILEREAD'] = profileread - else: - del os.environ['PROFILEREAD'] - - self.assertEqual(out, "hello\n") - self.assertEqual(ec, 0) - run_log.setLevel(run_log_level) - def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/sandbox/easybuild/easyblocks/foo.py b/test/framework/sandbox/easybuild/easyblocks/foo.py index 769f2a7e5a..8f1145fe72 100644 --- a/test/framework/sandbox/easybuild/easyblocks/foo.py +++ b/test/framework/sandbox/easybuild/easyblocks/foo.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/foofoo.py b/test/framework/sandbox/easybuild/easyblocks/foofoo.py index 511953a0fe..94ec86ef89 100644 --- a/test/framework/sandbox/easybuild/easyblocks/foofoo.py +++ b/test/framework/sandbox/easybuild/easyblocks/foofoo.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/bar.py b/test/framework/sandbox/easybuild/easyblocks/generic/bar.py index 2dcc4c5c01..6d62238e39 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/bar.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/bar.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py b/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py index 8304b9e42c..7fba52ca5b 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py index 8024520f32..59d96675e1 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py b/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py index 7ba202cbdc..f8212fdf96 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index b6e15d6347..f41a63c580 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/hpl.py b/test/framework/sandbox/easybuild/easyblocks/hpl.py index 1ef029f6c1..ba6bfe42f7 100644 --- a/test/framework/sandbox/easybuild/easyblocks/hpl.py +++ b/test/framework/sandbox/easybuild/easyblocks/hpl.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/scalapack.py b/test/framework/sandbox/easybuild/easyblocks/scalapack.py index 4534a5cd10..3ed062e7d0 100644 --- a/test/framework/sandbox/easybuild/easyblocks/scalapack.py +++ b/test/framework/sandbox/easybuild/easyblocks/scalapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/toy.py b/test/framework/sandbox/easybuild/easyblocks/toy.py index b03b6d9959..43d9eebe1c 100644 --- a/test/framework/sandbox/easybuild/easyblocks/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/toy.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -33,10 +33,12 @@ import shutil from easybuild.framework.easyblock import EasyBlock +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import mkdir from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.run import run_cmd + class EB_toy(EasyBlock): """Support for building/installing toy.""" @@ -55,7 +57,7 @@ def configure_step(self, name=None): # make sure Python system dep is handled correctly when specified if self.cfg['allow_system_deps']: if get_software_root('Python') != 'Python' or get_software_version('Python') != platform.python_version(): - self.log.error("Sanity check on allowed Python system dep failed.") + raise EasyBlock("Sanity check on allowed Python system dep failed.") os.rename('%s.source' % name, '%s.c' % name) def build_step(self, name=None): diff --git a/test/framework/sandbox/easybuild/easyblocks/toy_buggy.py b/test/framework/sandbox/easybuild/easyblocks/toy_buggy.py index 07658b64fd..74df658f3d 100644 --- a/test/framework/sandbox/easybuild/easyblocks/toy_buggy.py +++ b/test/framework/sandbox/easybuild/easyblocks/toy_buggy.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/__init__.py b/test/framework/sandbox/easybuild/tools/__init__.py index 15512e2f9c..6badf9fb34 100644 --- a/test/framework/sandbox/easybuild/tools/__init__.py +++ b/test/framework/sandbox/easybuild/tools/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2014 Ghent University +# Copyright 2009-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py index 1a66de17e0..5a2060c2d0 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2014 Ghent University +# Copyright 2011-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py new file mode 100644 index 0000000000..13c1634b3d --- /dev/null +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py @@ -0,0 +1,37 @@ +## +# Copyright 2013-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Implementation of a test module naming scheme that can be used to migrate from EasyBuildMNS to HierarchicalMNS. + +@author: Kenneth Hoste (Ghent University) +""" +from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS +from easybuild.tools.module_naming_scheme.hierarchical_mns import HierarchicalMNS + +class MigrateFromEBToHMNS(HierarchicalMNS, EasyBuildMNS): + + def det_install_subdir(self, ec): + """Determine name of software installation subdirectory of install path, using EasyBuild MNS.""" + return EasyBuildMNS.det_full_module_name(self, ec) diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py index 1fa5865941..ccd03960ca 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py index 8e8d74e0c8..b8b95a9dda 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/scripts.py b/test/framework/scripts.py index 3b78519baa..f406640da0 100644 --- a/test/framework/scripts.py +++ b/test/framework/scripts.py @@ -34,6 +34,9 @@ from test.framework.utilities import EnhancedTestCase from unittest import TestLoader, main +import vsc + +import easybuild.framework from easybuild.framework.easyconfig.easyconfig import EasyConfig from easybuild.tools.filetools import read_file, write_file from easybuild.tools.run import run_cmd @@ -42,6 +45,16 @@ class ScriptsTest(EnhancedTestCase): """ Testcase for run module """ + def setUp(self): + """Test setup.""" + super(ScriptsTest, self).setUp() + + # make sure both vsc-base and easybuild-framework are included in $PYTHONPATH (so scripts can pick it up) + vsc_loc = os.path.dirname(os.path.dirname(os.path.abspath(vsc.__file__))) + framework_loc = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(easybuild.framework.__file__)))) + pythonpath = os.environ.get('PYTHONPATH', '') + os.environ['PYTHONPATH'] = os.pathsep.join([vsc_loc, framework_loc, pythonpath]) + def test_generate_software_list(self): """Test for generate_software_list.py script.""" diff --git a/test/framework/suite.py b/test/framework/suite.py index f854208879..2b6001221f 100755 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -1,6 +1,6 @@ #!/usr/bin/python # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -54,6 +54,7 @@ # toolkit should be first to allow hacks to work import test.framework.asyncprocess as a +import test.framework.build_log as bl import test.framework.config as c import test.framework.easyblock as b import test.framework.easyconfig as e @@ -99,7 +100,7 @@ # call suite() for each module and then run them all # note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config -tests = [gen, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, l, f_c, sc, tw, p] +tests = [gen, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, l, f_c, sc, tw, p] SUITE = unittest.TestSuite([x.suite() for x in tests]) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index c33319e96a..0bda1e2e2c 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 01b6ce5769..e5f1d9890f 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -39,6 +39,7 @@ from easybuild.framework.easyconfig.easyconfig import EasyConfig, ActiveMNS from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import write_file +from easybuild.tools.modules import modules_tool from easybuild.tools.toolchain.utilities import search_toolchain from test.framework.utilities import find_full_path @@ -543,6 +544,79 @@ def test_mpi_cmd_for(self): shutil.rmtree(tmpdir) write_file(imkl_module_path, imkl_module_txt) + def test_prepare_deps(self): + """Test preparing for a toolchain when dependencies are involved.""" + tc = self.get_toolchain('GCC', version='4.6.4') + deps = [ + { + 'name': 'OpenMPI', + 'version': '1.6.4', + 'full_mod_name': 'OpenMPI/1.6.4-GCC-4.6.4', + 'short_mod_name': 'OpenMPI/1.6.4-GCC-4.6.4', + 'external_module': False, + }, + ] + tc.add_dependencies(deps) + tc.prepare() + mods = ['GCC/4.6.4', 'hwloc/1.6.2-GCC-4.6.4', 'OpenMPI/1.6.4-GCC-4.6.4'] + self.assertTrue([m['mod_name'] for m in modules_tool().list()], mods) + + def test_prepare_deps_external(self): + """Test preparing for a toolchain when dependencies and external modules are involved.""" + deps = [ + { + 'name': 'OpenMPI', + 'version': '1.6.4', + 'full_mod_name': 'OpenMPI/1.6.4-GCC-4.6.4', + 'short_mod_name': 'OpenMPI/1.6.4-GCC-4.6.4', + 'external_module': False, + 'external_module_metadata': {}, + }, + # no metadata available + { + 'name': None, + 'version': None, + 'full_mod_name': 'toy/0.0', + 'short_mod_name': 'toy/0.0', + 'external_module': True, + 'external_module_metadata': {}, + } + ] + tc = self.get_toolchain('GCC', version='4.6.4') + tc.add_dependencies(deps) + tc.prepare() + mods = ['GCC/4.6.4', 'hwloc/1.6.2-GCC-4.6.4', 'OpenMPI/1.6.4-GCC-4.6.4', 'toy/0.0'] + self.assertTrue([m['mod_name'] for m in modules_tool().list()], mods) + self.assertTrue(os.environ['EBROOTTOY'].endswith('software/toy/0.0')) + self.assertEqual(os.environ['EBVERSIONTOY'], '0.0') + self.assertFalse('EBROOTFOOBAR' in os.environ) + + # with metadata + deps[1] = { + 'full_mod_name': 'toy/0.0', + 'short_mod_name': 'toy/0.0', + 'external_module': True, + 'external_module_metadata': { + 'name': ['toy', 'foobar'], + 'version': ['1.2.3', '4.5'], + 'prefix': 'FOOBAR_PREFIX', + } + } + tc = self.get_toolchain('GCC', version='4.6.4') + tc.add_dependencies(deps) + os.environ['FOOBAR_PREFIX'] = '/foo/bar' + tc.prepare() + mods = ['GCC/4.6.4', 'hwloc/1.6.2-GCC-4.6.4', 'OpenMPI/1.6.4-GCC-4.6.4', 'toy/0.0'] + self.assertTrue([m['mod_name'] for m in modules_tool().list()], mods) + self.assertEqual(os.environ['EBROOTTOY'], '/foo/bar') + self.assertEqual(os.environ['EBVERSIONTOY'], '1.2.3') + self.assertEqual(os.environ['EBROOTFOOBAR'], '/foo/bar') + self.assertEqual(os.environ['EBVERSIONFOOBAR'], '4.5') + + self.assertEqual(modules.get_software_root('foobar'), '/foo/bar') + self.assertEqual(modules.get_software_version('toy'), '1.2.3') + + def suite(): """ return all the tests""" return TestLoader().loadTestsFromTestCase(ToolchainTest) diff --git a/test/framework/toolchainvariables.py b/test/framework/toolchainvariables.py index 9d7b996b33..5a9f8c369e 100644 --- a/test/framework/toolchainvariables.py +++ b/test/framework/toolchainvariables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 5d0aef6add..46797db2d1 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2014 Ghent University +# Copyright 2013-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -43,7 +43,8 @@ import easybuild.tools.module_naming_scheme # required to dynamically load test module naming scheme(s) from easybuild.framework.easyconfig.easyconfig import EasyConfig from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import mkdir, read_file, write_file +from easybuild.tools.config import get_module_syntax +from easybuild.tools.filetools import mkdir, read_file, which, write_file from easybuild.tools.modules import modules_tool @@ -79,10 +80,14 @@ def check_toy(self, installpath, outtxt, version='0.0', versionprefix='', versio # if the module exists, it should be fine toy_module = os.path.join(installpath, 'modules', 'all', 'toy', full_version) msg = "module for toy build toy/%s found (path %s)" % (full_version, toy_module) + if get_module_syntax() == 'Lua': + toy_module += '.lua' self.assertTrue(os.path.exists(toy_module), msg) # module file is symlinked according to moduleclass toy_module_symlink = os.path.join(installpath, 'modules', 'tools', 'toy', full_version) + if get_module_syntax() == 'Lua': + toy_module_symlink += '.lua' self.assertTrue(os.path.islink(toy_module_symlink)) self.assertTrue(os.path.exists(toy_module_symlink)) @@ -180,11 +185,11 @@ def test_toy_broken(self): verify=False, fails=True, verbose=False, raise_error=True) # make sure log file is retained, also for failed build - log_path_pattern = os.path.join(tmpdir, 'easybuild-*', 'easybuild-toy-0.0*.log') + log_path_pattern = os.path.join(tmpdir, 'eb-*', 'easybuild-toy-0.0*.log') self.assertTrue(len(glob.glob(log_path_pattern)) == 1, "Log file found at %s" % log_path_pattern) # make sure individual test report is retained, also for failed build - test_report_fp_pattern = os.path.join(tmpdir, 'easybuild-*', 'easybuild-toy-0.0*test_report.md') + test_report_fp_pattern = os.path.join(tmpdir, 'eb-*', 'easybuild-toy-0.0*test_report.md') self.assertTrue(len(glob.glob(test_report_fp_pattern)) == 1, "Test report %s found" % test_report_fp_pattern) # test dumping full test report (doesn't raise an exception) @@ -204,10 +209,11 @@ def test_toy_tweaked(self): # tweak easyconfig by appending to it ec_extra = '\n'.join([ "versionsuffix = '-tweaked'", - "modextrapaths = {'SOMEPATH': ['foo/bar', 'baz']}", + "modextrapaths = {'SOMEPATH': ['foo/bar', 'baz', '']}", "modextravars = {'FOO': 'bar'}", "modloadmsg = 'THANKS FOR LOADING ME, I AM %(name)s v%(version)s'", - "modtclfooter = 'puts stderr \"oh hai!\"'", + "modtclfooter = 'puts stderr \"oh hai!\"'", # ignored when module syntax is Lua + "modluafooter = 'io.stderr:write(\"oh hai!\")'" # ignored when module syntax is Tcl ]) write_file(ec_file, ec_extra, append=True) @@ -222,13 +228,27 @@ def test_toy_tweaked(self): outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True) self.check_toy(self.test_installpath, outtxt, versionsuffix='-tweaked') toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-tweaked') + if get_module_syntax() == 'Lua': + toy_module += '.lua' toy_module_txt = read_file(toy_module) - self.assertTrue(re.search('setenv\s*FOO\s*"bar"', toy_module_txt)) - self.assertTrue(re.search('prepend-path\s*SOMEPATH\s*\$root/foo/bar', toy_module_txt)) - self.assertTrue(re.search('prepend-path\s*SOMEPATH\s*\$root/baz', toy_module_txt)) - self.assertTrue(re.search('module-info mode load.*\n\s*puts stderr\s*.*I AM toy v0.0', toy_module_txt)) - self.assertTrue(re.search('puts stderr "oh hai!"', toy_module_txt)) + if get_module_syntax() == 'Tcl': + self.assertTrue(re.search(r'^setenv\s*FOO\s*"bar"$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root/foo/bar$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root/baz$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'module-info mode load.*\n\s*puts stderr\s*.*I AM toy v0.0"$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^puts stderr "oh hai!"$', toy_module_txt, re.M)) + elif get_module_syntax() == 'Lua': + self.assertTrue(re.search(r'^setenv\("FOO", "bar"\)', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", pathJoin\(root, "foo/bar"\)\)$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", pathJoin\(root, "baz"\)\)$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", root\)$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^if mode\(\) == "load" then\n\s*io.stderr:write\(".*I AM toy v0.0"\)$', + toy_module_txt, re.M)) + self.assertTrue(re.search(r'^io.stderr:write\("oh hai!"\)$', toy_module_txt, re.M)) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) def test_toy_buggy_easyblock(self): """Test build using a buggy/broken easyblock, make sure a traceback is reported.""" @@ -240,7 +260,7 @@ def test_toy_buggy_easyblock(self): 'verify': False, 'verbose': False, } - err_regex = r"crashed with an error.*Traceback[\S\s]*toy_buggy.py.*build_step[\S\s]*global name 'run_cmd'" + err_regex = r"Traceback[\S\s]*toy_buggy.py.*build_step[\S\s]*global name 'run_cmd'" self.assertErrorRegex(EasyBuildError, err_regex, self.test_toy_build, **kwargs) def test_toy_build_formatv2(self): @@ -442,8 +462,12 @@ def test_toy_permissions(self): (('modules', ), dir_perms), (('modules', 'all'), dir_perms), (('modules', 'all', 'toy'), dir_perms), - (('modules', 'all', 'toy', '0.0'), fil_perms), ]) + if get_module_syntax() == 'Tcl': + paths_perms.append((('modules', 'all', 'toy', '0.0'), fil_perms)) + elif get_module_syntax() == 'Lua': + paths_perms.append((('modules', 'all', 'toy', '0.0.lua'), fil_perms)) + for path, correct_perms in paths_perms: fullpath = glob.glob(os.path.join(self.test_installpath, *path))[0] perms = os.stat(fullpath).st_mode & 0777 @@ -532,16 +556,25 @@ def test_toy_hierarchical(self): # make sure module file is installed in correct path toy_module_path = os.path.join(mod_prefix, 'MPI', 'GCC', '4.7.2', 'OpenMPI', '1.6.4', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_module_path += '.lua' self.assertTrue(os.path.exists(toy_module_path)) # check that toolchain load is expanded to loads for toolchain dependencies, # except for the ones that extend $MODULEPATH to make the toy module available + if get_module_syntax() == 'Tcl': + load_regex_template = "load %s" + elif get_module_syntax() == 'Lua': + load_regex_template = r'load\("%s/.*"\)' + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + modtxt = read_file(toy_module_path) for dep in ['goolf', 'GCC', 'OpenMPI']: - load_regex = re.compile("load %s" % dep) + load_regex = re.compile(load_regex_template % dep) self.assertFalse(load_regex.search(modtxt), "Pattern '%s' not found in %s" % (load_regex.pattern, modtxt)) for dep in ['OpenBLAS', 'FFTW', 'ScaLAPACK']: - load_regex = re.compile("load %s" % dep) + load_regex = re.compile(load_regex_template % dep) self.assertTrue(load_regex.search(modtxt), "Pattern '%s' found in %s" % (load_regex.pattern, modtxt)) os.remove(toy_module_path) @@ -554,6 +587,8 @@ def test_toy_hierarchical(self): # make sure module file is installed in correct path toy_module_path = os.path.join(mod_prefix, 'Compiler', 'GCC', '4.7.2', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_module_path += '.lua' self.assertTrue(os.path.exists(toy_module_path)) # no dependencies or toolchain => no module load statements in module file @@ -570,12 +605,21 @@ def test_toy_hierarchical(self): # make sure module file is installed in correct path toy_module_path = os.path.join(mod_prefix, 'Compiler', 'GCC', '4.7.2', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_module_path += '.lua' self.assertTrue(os.path.exists(toy_module_path)) # 'module use' statements to extend $MODULEPATH are present modtxt = read_file(toy_module_path) modpath_extension = os.path.join(mod_prefix, 'MPI', 'GCC', '4.7.2', 'toy', '0.0') - self.assertTrue(re.search("^module\s*use\s*%s" % modpath_extension, modtxt, re.M)) + if get_module_syntax() == 'Tcl': + self.assertTrue(re.search("^module\s*use\s*%s" % modpath_extension, modtxt, re.M)) + elif get_module_syntax() == 'Lua': + fullmodpath_extension = os.path.join(self.test_installpath, modpath_extension) + regex = re.compile(r'^prepend_path\("MODULEPATH", "%s"\)' % fullmodpath_extension, re.M) + self.assertTrue(regex.search(modtxt), "Pattern '%s' found in %s" % (regex.pattern, modtxt)) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) os.remove(toy_module_path) # ... unless they shouldn't be @@ -583,7 +627,14 @@ def test_toy_hierarchical(self): self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True) modtxt = read_file(toy_module_path) modpath_extension = os.path.join(mod_prefix, 'MPI', 'GCC', '4.7.2', 'toy', '0.0') - self.assertFalse(re.search("^module\s*use\s*%s" % modpath_extension, modtxt, re.M)) + if get_module_syntax() == 'Tcl': + self.assertFalse(re.search("^module\s*use\s*%s" % modpath_extension, modtxt, re.M)) + elif get_module_syntax() == 'Lua': + fullmodpath_extension = os.path.join(self.test_installpath, modpath_extension) + regex = re.compile(r'^prepend_path\("MODULEPATH", "%s"\)' % fullmodpath_extension, re.M) + self.assertFalse(regex.search(modtxt), "Pattern '%s' found in %s" % (regex.pattern, modtxt)) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) os.remove(toy_module_path) # test module path with dummy/dummy build @@ -594,6 +645,8 @@ def test_toy_hierarchical(self): # make sure module file is installed in correct path toy_module_path = os.path.join(mod_prefix, 'Core', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_module_path += '.lua' self.assertTrue(os.path.exists(toy_module_path)) # no dependencies or toolchain => no module load statements in module file @@ -610,12 +663,21 @@ def test_toy_hierarchical(self): # make sure module file is installed in correct path toy_module_path = os.path.join(mod_prefix, 'Core', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_module_path += '.lua' self.assertTrue(os.path.exists(toy_module_path)) # no dependencies or toolchain => no module load statements in module file modtxt = read_file(toy_module_path) modpath_extension = os.path.join(mod_prefix, 'Compiler', 'toy', '0.0') - self.assertTrue(re.search(r"^module\s*use\s*%s" % modpath_extension, modtxt, re.M)) + if get_module_syntax() == 'Tcl': + self.assertTrue(re.search("^module\s*use\s*%s" % modpath_extension, modtxt, re.M)) + elif get_module_syntax() == 'Lua': + fullmodpath_extension = os.path.join(self.test_installpath, modpath_extension) + regex = re.compile(r'^prepend_path\("MODULEPATH", "%s"\)' % fullmodpath_extension, re.M) + self.assertTrue(regex.search(modtxt), "Pattern '%s' found in %s" % (regex.pattern, modtxt)) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) os.remove(toy_module_path) # building a toolchain module should also work @@ -623,6 +685,8 @@ def test_toy_hierarchical(self): modules_tool().purge() self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=False) gompi_module_path = os.path.join(mod_prefix, 'Core', 'gompi', '1.4.10') + if get_module_syntax() == 'Lua': + gompi_module_path += '.lua' self.assertTrue(os.path.exists(gompi_module_path)) def test_toy_advanced(self): @@ -638,6 +702,8 @@ def test_toy_hidden(self): self.test_toy_build(ec_file=ec_file, extra_args=['--hidden'], verify=False) # module file is hidden toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '.0.0') + if get_module_syntax() == 'Lua': + toy_module += '.lua' self.assertTrue(os.path.exists(toy_module), 'Found hidden module %s' % toy_module) # installed software is not hidden toybin = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin', 'toy') @@ -667,11 +733,15 @@ def test_module_filepath_tweaking(self): ] self.eb_main(args, do_build=True, verbose=True) mod_file_prefix = os.path.join(self.test_installpath, 'modules') - self.assertTrue(os.path.exists(os.path.join(mod_file_prefix, 'foobarbaz', 'toy', '0.0'))) - self.assertTrue(os.path.exists(os.path.join(mod_file_prefix, 'TOOLS', 'toy', '0.0'))) - self.assertTrue(os.path.islink(os.path.join(mod_file_prefix, 'TOOLS', 'toy', '0.0'))) - self.assertTrue(os.path.exists(os.path.join(mod_file_prefix, 't', 'toy', '0.0'))) - self.assertTrue(os.path.islink(os.path.join(mod_file_prefix, 't', 'toy', '0.0'))) + mod_file_suffix = '' + if get_module_syntax() == 'Lua': + mod_file_suffix += '.lua' + + self.assertTrue(os.path.exists(os.path.join(mod_file_prefix, 'foobarbaz', 'toy', '0.0' + mod_file_suffix))) + self.assertTrue(os.path.exists(os.path.join(mod_file_prefix, 'TOOLS', 'toy', '0.0' + mod_file_suffix))) + self.assertTrue(os.path.islink(os.path.join(mod_file_prefix, 'TOOLS', 'toy', '0.0' + mod_file_suffix))) + self.assertTrue(os.path.exists(os.path.join(mod_file_prefix, 't', 'toy', '0.0' + mod_file_suffix))) + self.assertTrue(os.path.islink(os.path.join(mod_file_prefix, 't', 'toy', '0.0' + mod_file_suffix))) def test_toy_archived_easyconfig(self): """Test archived easyconfig for a succesful build.""" @@ -688,6 +758,218 @@ def test_toy_archived_easyconfig(self): self.assertEqual(ec.name, 'toy') self.assertEqual(ec.version, '0.0') + def test_toy_module_fulltxt(self): + """Strict text comparison of generated module file.""" + self.test_toy_tweaked() + + toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-tweaked') + if get_module_syntax() == 'Lua': + toy_module += '.lua' + toy_mod_txt = read_file(toy_module) + + if get_module_syntax() == 'Lua': + mod_txt_regex_pattern = '\n'.join([ + r'help\(\[\[Toy C program. - Homepage: http://hpcugent.github.com/easybuild\]\]\)', + r'whatis\(\[\[Name: toy\]\]\)', + r'whatis\(\[\[Version: 0.0\]\]\)', + r'whatis\(\[\[Description: Toy C program. - Homepage: http://hpcugent.github.com/easybuild\]\]\)', + r'whatis\(\[\[Homepage: http://hpcugent.github.com/easybuild\]\]\)', + r'', + r'local root = "%s/software/toy/0.0-tweaked"' % self.test_installpath, + r'', + r'conflict\("toy"\)', + r'', + r'prepend_path\("LD_LIBRARY_PATH", pathJoin\(root, "lib"\)\)', + r'prepend_path\("LIBRARY_PATH", pathJoin\(root, "lib"\)\)', + r'prepend_path\("PATH", pathJoin\(root, "bin"\)\)', + r'setenv\("EBROOTTOY", root\)', + r'setenv\("EBVERSIONTOY", "0.0"\)', + r'setenv\("EBDEVELTOY", pathJoin\(root, "easybuild/toy-0.0-tweaked-easybuild-devel"\)\)', + r'', + r'setenv\("FOO", "bar"\)', + r'prepend_path\("SOMEPATH", pathJoin\(root, "foo/bar"\)\)', + r'prepend_path\("SOMEPATH", pathJoin\(root, "baz"\)\)', + r'prepend_path\("SOMEPATH", root\)', + r'', + r'if mode\(\) == "load" then', + r' io.stderr:write\("THANKS FOR LOADING ME, I AM toy v0.0"\)', + r'end', + r'io.stderr:write\("oh hai\!"\)', + r'-- Built with EasyBuild version .*$', + ]) + elif get_module_syntax() == 'Tcl': + mod_txt_regex_pattern = '\n'.join([ + r'^#%Module', + r'proc ModulesHelp { } {', + r' puts stderr { Toy C program. - Homepage: http://hpcugent.github.com/easybuild', + r' }', + r'}', + r'', + r'module-whatis {Description: Toy C program. - Homepage: http://hpcugent.github.com/easybuild}', + r'', + r'set root %s/software/toy/0.0-tweaked' % self.test_installpath, + r'', + r'conflict toy', + r'', + r'prepend-path LD_LIBRARY_PATH \$root/lib', + r'prepend-path LIBRARY_PATH \$root/lib', + r'prepend-path PATH \$root/bin', + r'setenv EBROOTTOY "\$root"', + r'setenv EBVERSIONTOY "0.0"', + r'setenv EBDEVELTOY "\$root/easybuild/toy-0.0-tweaked-easybuild-devel"', + r'', + r'setenv FOO "bar"', + r'prepend-path SOMEPATH \$root/foo/bar', + r'prepend-path SOMEPATH \$root/baz', + r'prepend-path SOMEPATH \$root', + r'', + r'if { \[ module-info mode load \] } {', + r' puts stderr "THANKS FOR LOADING ME, I AM toy v0.0"', + r'}', + r'puts stderr "oh hai\!"', + r'# Built with EasyBuild version .*$', + ]) + else: + self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + + mod_txt_regex = re.compile(mod_txt_regex_pattern) + msg = "Pattern '%s' matches with: %s" % (mod_txt_regex.pattern, toy_mod_txt) + self.assertTrue(mod_txt_regex.match(toy_mod_txt), msg) + + def test_external_dependencies(self): + """Test specifying external (build) dependencies.""" + ectxt = read_file(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'toy-0.0-deps.eb')) + toy_ec = os.path.join(self.test_prefix, 'toy-0.0-external-deps.eb') + + # just specify some of the test modules we ship, doesn't matter where they come from + extraectxt = "\ndependencies += [('foobar/1.2.3', EXTERNAL_MODULE)]" + extraectxt += "\nbuilddependencies = [('somebuilddep/0.1', EXTERNAL_MODULE)]" + extraectxt += "\nversionsuffix = '-external-deps'" + write_file(toy_ec, ectxt + extraectxt) + + # install dummy modules + modulepath = os.path.join(self.test_prefix, 'modules') + for mod in ['ictce/4.1.13', 'GCC/4.7.2', 'foobar/1.2.3', 'somebuilddep/0.1']: + mkdir(os.path.join(modulepath, os.path.dirname(mod)), parents=True) + write_file(os.path.join(modulepath, mod), "#%Module") + + self.reset_modulepath([modulepath]) + self.test_toy_build(ec_file=toy_ec, versionsuffix='-external-deps', verbose=True) + + modules_tool().load(['toy/0.0-external-deps']) + # note build dependency is not loaded + mods = ['ictce/4.1.13', 'GCC/4.7.2', 'foobar/1.2.3', 'toy/0.0-external-deps'] + self.assertEqual([x['mod_name'] for x in modules_tool().list()], mods) + + # check behaviour when a non-existing external (build) dependency is included + err_msg = "Missing modules for one or more dependencies marked as external modules:" + + extraectxt = "\nbuilddependencies = [('nosuchbuilddep/0.0.0', EXTERNAL_MODULE)]" + extraectxt += "\nversionsuffix = '-external-deps-broken1'" + write_file(toy_ec, ectxt + extraectxt) + self.assertErrorRegex(EasyBuildError, err_msg, self.test_toy_build, ec_file=toy_ec, + raise_error=True, verbose=False) + + extraectxt = "\ndependencies += [('nosuchmodule/1.2.3', EXTERNAL_MODULE)]" + extraectxt += "\nversionsuffix = '-external-deps-broken2'" + write_file(toy_ec, ectxt + extraectxt) + self.assertErrorRegex(EasyBuildError, err_msg, self.test_toy_build, ec_file=toy_ec, + raise_error=True, verbose=False) + + # --dry-run still works when external modules are missing; external modules are treated as if they were there + outtxt = self.test_toy_build(ec_file=toy_ec, verbose=True, extra_args=['--dry-run'], verify=False) + self.assertTrue(re.search(r"^ \* \[ \] .* \(module: toy/0.0-external-deps-broken2\)", outtxt, re.M)) + + def test_module_only(self): + """Test use of --module-only.""" + ec_files_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + ec_file = os.path.join(ec_files_path, 'toy-0.0-deps.eb') + toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-deps') + + # only consider provided test modules + self.reset_modulepath([os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules')]) + + # sanity check fails without --force if software is not installed yet + common_args = [ + ec_file, + '--sourcepath=%s' % self.test_sourcepath, + '--buildpath=%s' % self.test_buildpath, + '--installpath=%s' % self.test_installpath, + '--debug', + '--unittest-file=%s' % self.logfile, + '--robot=%s' % ec_files_path, + '--module-syntax=Tcl', + ] + args = common_args + ['--module-only'] + err_msg = "Sanity check failed" + self.assertErrorRegex(EasyBuildError, err_msg, self.eb_main, args, do_build=True, raise_error=True) + self.assertFalse(os.path.exists(toy_mod)) + + self.eb_main(args + ['--force'], do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_mod)) + + # make sure load statements for dependencies are included in additional module file generated with --module-only + modtxt = read_file(toy_mod) + self.assertTrue(re.search('load.*ictce/4.1.13', modtxt), "load statement for ictce/4.1.13 found in module") + self.assertTrue(re.search('load.*GCC/4.7.2', modtxt), "load statement for GCC/4.7.2 found in module") + + os.remove(toy_mod) + + # installing another module under a different naming scheme and using Lua module syntax works fine + + # first actually build and install toy software + module + prefix = os.path.join(self.test_installpath, 'software', 'toy', '0.0-deps') + self.eb_main(common_args + ['--force'], do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_mod)) + self.assertTrue(os.path.exists(os.path.join(self.test_installpath, 'software', 'toy', '0.0-deps', 'bin'))) + modtxt = read_file(toy_mod) + self.assertTrue(re.search("set root %s" % prefix, modtxt)) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 1) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1) + + # install (only) additional module under a hierarchical MNS + args = common_args + [ + '--module-only', + '--module-naming-scheme=MigrateFromEBToHMNS', + ] + toy_core_mod = os.path.join(self.test_installpath, 'modules', 'all', 'Core', 'toy', '0.0-deps') + self.assertFalse(os.path.exists(toy_core_mod)) + self.eb_main(args, do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_core_mod)) + # existing install is reused + modtxt2 = read_file(toy_core_mod) + self.assertTrue(re.search("set root %s" % prefix, modtxt2)) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 2) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1) + + # make sure load statements for dependencies are included + modtxt = read_file(toy_core_mod) + self.assertTrue(re.search('load.*ictce/4.1.13', modtxt), "load statement for ictce/4.1.13 found in module") + + os.remove(toy_mod) + os.remove(toy_core_mod) + + # test installing (only) additional module in Lua syntax (if Lmod is available) + lmod_abspath = which('lmod') + if lmod_abspath is not None: + args = common_args[:-1] + [ + '--module-only', + '--module-syntax=Lua', + '--modules-tool=Lmod', + ] + self.assertFalse(os.path.exists(toy_mod + '.lua')) + self.eb_main(args, do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_mod + '.lua')) + # existing install is reused + modtxt3 = read_file(toy_mod + '.lua') + self.assertTrue(re.search('local root = "%s"' % prefix, modtxt3)) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 2) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1) + + # make sure load statements for dependencies are included + modtxt = read_file(toy_mod + '.lua') + self.assertTrue(re.search('load.*ictce/4.1.13', modtxt), "load statement for ictce/4.1.13 found in module") + def suite(): """ return all the tests in this file """ return TestLoader().loadTestsFromTestCase(ToyBuildTest) diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 5754c71760..4c24d5ec9c 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -46,7 +46,7 @@ from easybuild.framework.easyblock import EasyBlock from easybuild.main import main from easybuild.tools import config -from easybuild.tools.config import module_classes +from easybuild.tools.config import module_classes, set_tmpdir from easybuild.tools.environment import modify_env from easybuild.tools.filetools import mkdir, read_file from easybuild.tools.module_naming_scheme import GENERAL_CLASS @@ -82,11 +82,19 @@ class EnhancedTestCase(_EnhancedTestCase): def setUp(self): """Set up testcase.""" super(EnhancedTestCase, self).setUp() + + # keep track of log handlers + log = fancylogger.getLogger(fname=False) + self.orig_log_handlers = log.handlers[:] + + self.orig_tmpdir = tempfile.gettempdir() + # use a subdirectory for this test (which we can clean up easily after the test completes) + self.test_prefix = set_tmpdir() + self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) fd, self.logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') os.close(fd) self.cwd = os.getcwd() - self.test_prefix = tempfile.mkdtemp() # keep track of original environment to restore self.orig_environ = copy.deepcopy(os.environ) @@ -94,10 +102,6 @@ def setUp(self): # keep track of original environment/Python search path to restore self.orig_sys_path = sys.path[:] - self.orig_paths = {} - for path in ['buildpath', 'installpath', 'sourcepath']: - self.orig_paths[path] = os.environ.get('EASYBUILD_%s' % path.upper(), None) - testdir = os.path.dirname(os.path.abspath(__file__)) self.test_sourcepath = os.path.join(testdir, 'sandbox', 'sources') @@ -141,30 +145,35 @@ def setUp(self): def tearDown(self): """Clean up after running testcase.""" super(EnhancedTestCase, self).tearDown() + + # go back to where we were before os.chdir(self.cwd) + + # restore original environment modify_env(os.environ, self.orig_environ) - tempfile.tempdir = None # restore original Python search path sys.path = self.orig_sys_path - # cleanup - for path in [self.logfile, self.test_buildpath, self.test_installpath, self.test_prefix]: - try: - if os.path.isdir(path): - shutil.rmtree(path) - else: - os.remove(path) - except OSError, err: - pass - - for path in ['buildpath', 'installpath', 'sourcepath']: - if self.orig_paths[path] is not None: - os.environ['EASYBUILD_%s' % path.upper()] = self.orig_paths[path] - else: - if 'EASYBUILD_%s' % path.upper() in os.environ: - del os.environ['EASYBUILD_%s' % path.upper()] - init_config() + # remove any log handlers that were added (so that log files can be effectively removed) + log = fancylogger.getLogger(fname=False) + new_log_handlers = [h for h in log.handlers if h not in self.orig_log_handlers] + for log_handler in new_log_handlers: + log_handler.close() + log.removeHandler(log_handler) + + # cleanup test tmp dir + try: + shutil.rmtree(self.test_prefix) + except (OSError, IOError): + pass + + # restore original 'parent' tmpdir + for var in ['TMPDIR', 'TEMP', 'TMP']: + os.environ[var] = self.orig_tmpdir + + # reset to make sure tempfile picks up new temporary directory to use + tempfile.tempdir = None def reset_modulepath(self, modpaths): """Reset $MODULEPATH with specified paths.""" @@ -178,7 +187,8 @@ def reset_modulepath(self, modpaths): for modpath in modpaths: modtool.add_module_path(modpath) - def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbose=False, raise_error=False): + def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbose=False, raise_error=False, + reset_env=True): """Helper method to call EasyBuild main function.""" cleanup() @@ -186,7 +196,11 @@ def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbos if logfile is None: logfile = self.logfile # clear log file - open(logfile, 'w').write('') + f = open(logfile, 'w') + f.write('') + f.close() + + env_before = copy.deepcopy(os.environ) try: main((args, logfile, do_build)) @@ -197,18 +211,26 @@ def eb_main(self, args, do_build=False, return_error=False, logfile=None, verbos if verbose: print "err: %s" % err + logtxt = read_file(logfile) + os.chdir(self.cwd) # make sure config is reinitialized init_config() + # restore environment to what it was before running main, + # changes may have been made by eb_main (e.g. $TMPDIR & co) + if reset_env: + modify_env(os.environ, env_before) + tempfile.tempdir = None + if myerr and raise_error: raise myerr if return_error: - return read_file(self.logfile), myerr + return logtxt, myerr else: - return read_file(self.logfile) + return logtxt def setup_hierarchical_modules(self): """Setup hierarchical modules to run tests on.""" diff --git a/test/framework/variables.py b/test/framework/variables.py index 8d337173ad..c377b507b6 100644 --- a/test/framework/variables.py +++ b/test/framework/variables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2014 Ghent University +# Copyright 2012-2015 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -78,6 +78,16 @@ class TestVariables(Variables): cmd = CommandFlagList(["gcc", "bar", "baz"]) self.assertEqual(str(cmd), "gcc -bar -baz") + def test_empty_variables(self): + """Test playing around with empty variables.""" + v = Variables() + v.nappend('FOO', []) + self.assertEqual(v['FOO'], []) + v.join('BAR', 'FOO') + self.assertEqual(v['BAR'], []) + v.join('FOOBAR', 'BAR') + self.assertEqual(v['FOOBAR'], []) + def suite(): """ return all the tests""" return TestLoader().loadTestsFromTestCase(VariablesTest)