diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 10649cddab..9d866e3a8c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -138,7 +138,7 @@ You might also want to look into [hub](https://github.com/defunkt/hub) for more ### Review process -A member of the EasyBuild team will then review your pull request, paying attention to what you're contributing, how you implemented it and [Code style](https://github.com/hpcugent/easybuild/wiki/Code-style). +A member of the EasyBuild team will then review your pull request, paying attention to what you're contributing, how you implemented it and [code style](http://easybuild.readthedocs.org/en/latest/Code_style.html). Most likely, some remarks will be made on your pull request. Note that this is nothing personal, we're just trying to keep the EasyBuild codebase as high quality as possible. Even when an EasyBuild team member makes changes, the same public review process is followed. diff --git a/README.rst b/README.rst index 2cd7efc455..5b375ff615 100644 --- a/README.rst +++ b/README.rst @@ -1,54 +1,57 @@ -Build status - *master branch (Python 2.4, Python 2.6, Python 2.7)* - -.. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python24/badge/icon - :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python24/ -.. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master/badge/icon - :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master/ -.. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python27/badge/icon - :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python27/ - -Build status - *develop branch (Python 2.4, Python 2.6, Python 2.7)* - -.. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python24/badge/icon - :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python24/ -.. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop/badge/icon - :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop/ -.. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python27/badge/icon - :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python27/ - -EasyBuild: building software with ease --------------------------------------- - -The easybuild-framework package is the basis for EasyBuild -(http://hpcugent.github.com/easybuild), a software build and -installation framework written in Python that allows you to install -software in a structured, repeatable and robust way. - -This package contains the EasyBuild framework that supports the -implementation and use of so-called easyblocks, that implement the -software install procedure for a particular (group of) software +.. image:: http://hpcugent.github.io/easybuild/images/easybuild_logo_small.png + :align: center + +`EasyBuild `_ is a software build +and installation framework that allows you to manage (scientific) software +on High Performance Computing (HPC) systems in an efficient way. + +The **easybuild-framework** package is the core of EasyBuild. It +supports the implementation and use of so-called easyblocks which +implement the software install procedure for a particular (group of) software package(s). -The code of the easybuild-framework package is hosted on GitHub, along +The EasyBuild documentation is available at http://easybuild.readthedocs.org/. + +The EasyBuild framework source code is hosted on GitHub, along with an issue tracker for bug reports and feature requests, see http://github.com/hpcugent/easybuild-framework. -The EasyBuild documentation is available on the GitHub wiki of the -easybuild meta-package, see -http://github.com/hpcugent/easybuild/wiki/Home. - -Related packages: -- easybuild-easyblocks -(http://pypi.python.org/pypi/easybuild-easyblocks): a collection of -easyblocks that implement support for building and installing (groups -of) software packages. - -- easybuild-easyconfigs -(http://pypi.python.org/pypi/easybuild-easyconfigs): a collection of -example easyconfig files that specify which software to build, and using -which build options; these easyconfigs will be well tested with the -latest compatible versions of the easybuild-framework and -easybuild-easyblocks packages. - -The code in the vsc directory originally comes from VSC-tools -(https://github.com/hpcugent/VSC-tools). +Related Python packages: + +* **easybuild-easyblocks** + + * a collection of easyblocks that implement support for building and installing (groups of) software packages + * GitHub repository: http://github.com/hpcugent/easybuild-easyblocks + * package on PyPi: https://pypi.python.org/pypi/easybuild-easyblocks + +* **easybuild-easyconfigs** + + * a collection of example easyconfig files that specify which software to build, + and using which build options; these easyconfigs will be well tested + with the latest compatible versions of the easybuild-framework and easybuild-easyblocks packages + * GitHub repository: http://github.com/hpcugent/easybuild-easyconfigs + * PyPi: https://pypi.python.org/pypi/easybuild-easyconfigs + +The code in the ``vsc`` directory originally comes from the *vsc-base* package +(https://github.com/hpcugent/vsc-base). + + +*Build status overview:* + +* **master** branch *(Python 2.4, Python 2.6, Python 2.7)* + + .. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python24/badge/icon + :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python24/ + .. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master/badge/icon + :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master/ + .. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python27/badge/icon + :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python27/ + +* **develop** branch *(Python 2.4, Python 2.6, Python 2.7)* + + .. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python24/badge/icon + :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python24/ + .. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop/badge/icon + :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop/ + .. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python27/badge/icon + :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python27/ diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 2186d01915..13ef17e9cc 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,6 +1,88 @@ This file contains a description of the major changes to the easybuild-framework EasyBuild package. For more detailed information, please see the git log. +These release notes can also be consulted at http://easybuild.readthedocs.org/en/latest/Release_notes.html. + +v1.16.1 (December 19th 2014) +---------------------------- + +bugfix release +- fix functionality that is broken with --deprecated=2.0 or with $EASYBUILD_DEPRECATED=2.0 + - don't include easyconfig parameters for ConfigureMake in eb -a, since fallback is deprecated (#1123) + - correctly check software_license value type (#1124) + - fix generate_software_list.py script w.r.t. deprecated fallback to ConfigureMake (#1127) +- other bug fixes + - fix logging issues in tests, sync with vsc-base v2.0.0 (#1120) + +v1.16.0 (December 18th 2014) +---------------------------- + +feature + bugfix release +- deprecate automagic fallback to ConfigureMake easyblock (#1113) + - easyconfigs should specify easyblock = 'ConfigureMake' instead of relying on the fallback mechanism + - note: automagic fallback to ConfigureMake easyblock will be dropped in EasyBuild v2.0 + - see also http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html#configuremake-fallback +- stop triggering deprecated functionality, to enable use of --deprecated=2.0 (#1107, #1115, #1119) + - see http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html#configuremake-fallback for more information +- various other enhancements, including: + - add script to clean up gists created via --upload-test-report (#958) + - also use -xHost when using Intel compilers on AMD systems (as opposed to -msse3) (#960) + - add Python version check in eb command (#1046) + - take versionprefix into account in HierarchicalMNS module naming scheme (#1058) + - clean up and refactor main.py, move functionality to other modules (#1059, #1064, #1075, #1087) + - add check in download_file function for HTTP return code + show download progress report (#1066, #1090) + - include info log message with name and location of used easyblock (#1069) + - add toolchains definitions for gpsmpi, gpsolf, impich, intel-para, ipsmpi toolchains (#1072, #1073) + - support for Parastation MPI based toolchains + - enforce that hiddendependencies is a subset of dependencies (#1078) + - this is done to avoid that site-specific policies w.r.t. hidden modules slip into contributed easyconfigs + - enable use of --show_hidden for avail subcommand with recent Lmod versions (#1081) + - add --robot-paths configure option (#1080, #1093, #1095, #1114) + - support use of %(DEFAULT_ROBOT_PATHS)s template in EasyBuild configuration files (#1100) + - see also http://easybuild.readthedocs.org/en/latest/Using_the_EasyBuild_command_line.html#controlling-the-robot-search-path + - use -xHost rather than -xHOST, to match Intel documentation (#1084) + - update and cleanup README file (#1085) + - deprecate self.moduleGenerator in favor of self.module_generator in EasyBlock (#1088) + - also support MPICH MPI family in mpi_cmd_for function (#1098) + - update documentation references to point to http://easybuild.readthedocs.org (#1102) + - check for OS dependencies with both rpm and dpkg (if available) (#1111) +- various bug fixes, including: + - fix picking required software version specified by --software-version and clean up tweak.py (#1062, #1063) + - escape $ characters in module load message specified via modloadmsg easyconfig parameter) (#1068) + - take available hidden modules into account in dependency resolution (#1065) + - fix hard crash when using patch files with an empty list of sources (#1070) + - fix Intel MKL BLACS library being used for MPICH/MPICH2-based toolchains (#1072) + - fix regular expression in fetch_parameter_from_easyconfig_file function (#1096) + - don’t hardcode queue names when submitting a job (#1106) + - fix affiliation/mail address for Fotis in headers (#1105) + - filter out /dev/null entries in patch file in det_patched_files function (#1108) + - fix gmpolf toolchain definition, to have gmpich as MPI components (instead of gmpich2) (#1101) + - ‘MPICH’ refers to MPICH v3.x, while MPICH2 refers to MPICH(2) v2.x (MPICH v1.x is ancient/obsolete) + - note: this requires to reinstall the gmpolf module, using the updated easyconfig from easybuild-easyconfigs#1217 + +v1.15.2 (October 7th 2014) +-------------------------- + +bugfix release +- fix $MODULEPATH extensions for Clang/CUDA, to make goolfc/cgoolf compatible with HierarchicalMNS (#1050) +- include versionsuffix in module subdirectory with HierarchicalMNS (#1050, #1055) +- fix unit tests which were broken with bash patched for ShellShock bug (#1051) +- add definition of gimpi toolchain, required to make gimkl toolchain compatible with HierarchicalMNS (#1052) +- don't override COMPILER_MODULE_NAME obtained from ClangGCC in Clang-based toolchains (#1053) +- fix wrong code in path_to_top_of_module_tree function (#1054) + - because of this, load statements for compilers were potentially included in higher-level modules under HierarchicalMNS + +v1.15.1 (September 23rd 2014) +----------------------------- + +bugfix release +- take into account that multiple modules may be extending $MODULEPATH with the same path, + when determining path to top of module tree (see #1047) + - this bug caused a load statement for either icc or ifort to be included in higher-level + modules installed with an Intel-based compiler toolchain, under the HierarchicalMNS module naming scheme +- make HierarchicalMNS module naming scheme compatible with cgoolf and goolfc toolchain (#1049) +- add definition of iompi (sub)toolchain to make iomkl toolchain compatible with HierarchicalMNS (#1049) + v1.15.0 (September 12th 2014) ----------------------------- diff --git a/easybuild/easybuild_config.py b/easybuild/easybuild_config.py deleted file mode 100644 index 0d90c875a1..0000000000 --- a/easybuild/easybuild_config.py +++ /dev/null @@ -1,96 +0,0 @@ -# # -# Copyright 2009-2014 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), -# the Hercules foundation (http://www.herculesstichting.be/in_English) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# http://github.com/hpcugent/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -# # -""" -EasyBuild configuration file. - This is now frozen. - All new configuration should be done through the options parser. - This is deprecated and will be removed in 2.0 - -@author: Stijn De Weirdt (Ghent University) -@author: Dries Verdegem (Ghent University) -@author: Kenneth Hoste (Ghent University) -@author: Pieter De Baets (Ghent University) -@author: Jens Timmerman (Ghent University) -@author: Toon Willems (Ghent University) -@author: Fotis Georgatos (University of Luxembourg) -""" - -# -# Developers, please do not add any new defaults or variables -# Use the config options -# - -import os -import tempfile - -import easybuild.tools.config as config - -# this should result in a MODULEPATH=($HOME/.local/easybuild|$EASYBUILDPREFIX)//all -if os.getenv('EASYBUILDPREFIX'): - prefix = os.getenv('EASYBUILDPREFIX') -else: - prefix = os.path.join(os.getenv('HOME'), ".local", "easybuild") - -# build/install/source paths configuration for EasyBuild -# build_path possibly overridden by EASYBUILDBUILDPATH -# install_path possibly overridden by EASYBUILDINSTALLPATH -build_path = os.path.join(prefix, 'build') -install_path = prefix -source_path = os.path.join(prefix, 'sources') - -# repository for eb files -# currently, EasyBuild supports the following repository types: - -# * `FileRepository`: a plain flat file repository. In this case, the `repositoryPath` contains the directory where the files are stored, -# * `GitRepository`: a _non-empty_ **bare** git repository (created with `git init --bare` or `git clone --bare`). -# Here, the `repositoryPath` contains the git repository location, which can be a directory or an URL. -# * `SvnRepository`: an SVN repository. In this case, the `repositoryPath` contains the subversion repository location, again, this can be a directory or an URL. - -# you have to set the `repository` variable inside the config like so: -# `repository = FileRepository(repositoryPath)` - -# optionally a subdir argument can be specified: -# `repository = FileRepository(repositoryPath, subdir)` -repository_path = os.path.join(prefix, 'ebfiles_repo') -repository = FileRepository(repository_path) # @UndefinedVariable (this file gets exec'ed, so ignore this) - -# log format: (dir, filename template) -# supported in template: name, version, data, time -log_format = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log") - -# set the path where log files will be stored -log_dir = tempfile.gettempdir() - -# define set of supported module classes -module_classes = ['base', 'bio', 'chem', 'compiler', 'lib', 'phys', 'tools', - 'cae', 'data', 'debugger', 'devel', 'ide', 'math', 'mpi', 'numlib', 'perf', 'system', 'vis'] - -# general cleanliness -del os, tempfile, config, prefix - -# -# Developers, please do not add any new defaults or variables -# Use the config options -# diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f1840258b4..4327d92fdd 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -33,12 +33,12 @@ @author: Jens Timmerman (Ghent University) @author: Toon Willems (Ghent University) @author: Ward Poelmans (Ghent University) -@author: Fotis Georgatos (University of Luxembourg) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ import copy import glob -import re +import inspect import os import shutil import stat @@ -46,18 +46,21 @@ import traceback from distutils.version import LooseVersion from vsc.utils import fancylogger +from vsc.utils.missing import get_class_for import easybuild.tools.environment as env from easybuild.tools import config, filetools -from easybuild.framework.easyconfig.easyconfig import (EasyConfig, ActiveMNS, ITERATE_OPTIONS, - fetch_parameter_from_easyconfig_file, get_class_for, get_easyblock_class, get_module_path, resolve_template) +from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR +from easybuild.framework.easyconfig.easyconfig import ITERATE_OPTIONS, EasyConfig, ActiveMNS +from easybuild.framework.easyconfig.easyconfig import fetch_parameter_from_easyconfig_file +from easybuild.framework.easyconfig.easyconfig import get_easyblock_class, get_module_path, resolve_template from easybuild.framework.easyconfig.tools import get_paths_for from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP from easybuild.tools.build_details import get_build_stats from easybuild.tools.build_log import EasyBuildError, print_error, print_msg from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath from easybuild.tools.config import install_path, log_path, read_only_installdir, source_paths -from easybuild.tools.environment import modify_env +from easybuild.tools.environment import restore_env from easybuild.tools.filetools import DEFAULT_CHECKSUM from easybuild.tools.filetools import adjust_permissions, apply_patch, convert_name, download_file, encode_class_name from easybuild.tools.filetools import extract_file, mkdir, read_file, rmtree2 @@ -74,6 +77,7 @@ from easybuild.tools.utilities import remove_unwanted_chars from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION + _log = fancylogger.getLogger('easyblock') @@ -93,15 +97,9 @@ def extra_options(extra=None): extra = {} if not isinstance(extra, dict): - _log.deprecated("Obtained value of type '%s' for extra, should be 'dict'" % type(extra), '2.0') - _log.debug("Converting extra_options value '%s' of type '%s' to a dict" % (extra, type(extra))) - extra = dict(extra) - - # to avoid breaking backward compatibility, we still need to return a list of tuples in EasyBuild v1.x - _log.deprecated("Returning list of tuples rather than a dict as return value of extra_options", '2.0') - res = extra.items() + _log.nosupport("Obtained 'extra' value of type '%s' in extra_options, should be 'dict'" % type(extra), '2.0') - return res + return extra # # INIT @@ -112,6 +110,9 @@ def __init__(self, ec): @param ec: a parsed easyconfig file (EasyConfig instance) """ + # keep track of original working directory, so we can go back there + self.orig_workdir = os.getcwd() + # list of patch/source files, along with checksums self.patches = [] self.src = [] @@ -131,7 +132,7 @@ def __init__(self, ec): # modules interface with default MODULEPATH self.modules_tool = modules_tool() # module generator - self.moduleGenerator = ModuleGeneratorLua(self, fake=True) + self.module_generator = ModuleGeneratorLua(self, fake=True) # modules footer self.modules_footer = None @@ -163,6 +164,12 @@ def __init__(self, ec): # list of loaded modules self.loaded_modules = [] + # iterate configure/build/options + self.iter_opts = {} + + # sanity check fail error messages to report (if any) + self.sanity_check_fail_msgs = [] + # robot path self.robot_path = build_option('robot_path') @@ -172,15 +179,9 @@ def __init__(self, ec): # keep track of original environment, so we restore it if needed self.orig_environ = copy.deepcopy(os.environ) - # at the end of __init__, initialise the logging + # initialize logger self._init_log() - # iterate configure/build/options - self.iter_opts = {} - - # sanity check fail error messages to report (if any) - self.sanity_check_fail_msgs = [] - # should we keep quiet? self.silent = build_option('silent') @@ -215,6 +216,10 @@ def _init_log(self): self.log.info(this_is_easybuild()) + this_module = inspect.getmodule(self) + tup = (self.__class__.__name__, this_module.__name__, this_module.__file__) + self.log.info("This is easyblock %s from module %s (%s)" % tup) + def close_log(self): """ Shutdown the logger. @@ -275,50 +280,55 @@ def fetch_sources(self, list_of_sources, checksums=None): self.log.info("Added sources: %s" % self.src) - def fetch_patches(self, list_of_patches, extension=False, checksums=None): + def fetch_patches(self, patch_specs=None, extension=False, checksums=None): """ Add a list of patches. All patches will be checked if a file exists (or can be located) """ + if patch_specs is None: + patch_specs = self.cfg['patches'] patches = [] - for index, patch_entry in enumerate(list_of_patches): + for index, patch_spec in enumerate(patch_specs): # check if the patches can be located copy_file = False suff = None level = None - if isinstance(patch_entry, (list, tuple)): - if not len(patch_entry) == 2: - self.log.error("Unknown patch specification '%s', only two-element lists/tuples are supported!" % patch_entry) - pf = patch_entry[0] - - if isinstance(patch_entry[1], int): - level = patch_entry[1] - elif isinstance(patch_entry[1], basestring): + if isinstance(patch_spec, (list, tuple)): + if not len(patch_spec) == 2: + self.log.error("Unknown patch specification '%s', only two-element lists/tuples are supported!", + str(patch_spec)) + patch_file = patch_spec[0] + + # this *must* be of typ int, nothing else + # no 'isinstance(..., int)', since that would make True/False also acceptable + if type(patch_spec[1]) == int: + level = patch_spec[1] + elif isinstance(patch_spec[1], basestring): # non-patch files are assumed to be files to copy - if not patch_entry[0].endswith('.patch'): + if not patch_spec[0].endswith('.patch'): copy_file = True - suff = patch_entry[1] + suff = patch_spec[1] else: - self.log.error("Wrong patch specification '%s', only int and string are supported as second element!" % patch_entry) + self.log.error("Wrong patch spec '%s', only int/string are supported as 2nd element" % str(patch_spec)) else: - pf = patch_entry + patch_file = patch_spec - path = self.obtain_file(pf, extension=extension) + path = self.obtain_file(patch_file, extension=extension) if path: - self.log.debug('File %s found for patch %s' % (path, patch_entry)) + self.log.debug('File %s found for patch %s' % (path, patch_spec)) patchspec = { - 'name': pf, + 'name': patch_file, 'path': path, - 'checksum': self.get_checksum_for(checksums, filename=pf, index=index), + 'checksum': self.get_checksum_for(checksums, filename=patch_file, index=index), } if suff: if copy_file: patchspec['copy'] = suff else: patchspec['sourcepath'] = suff - if level: + if level is not None: patchspec['level'] = level if extension: @@ -326,7 +336,7 @@ def fetch_patches(self, list_of_patches, extension=False, checksums=None): else: self.patches.append(patchspec) else: - self.log.error('No file found for patch %s' % patch_entry) + self.log.error('No file found for patch %s' % patch_spec) if extension: self.log.info("Fetched extension patches: %s" % patches) @@ -393,7 +403,7 @@ def fetch_extension_sources(self): else: self.log.error('Checksum for ext source %s failed' % fn) - ext_patches = self.fetch_patches(ext_options.get('patches', []), extension=True) + ext_patches = self.fetch_patches(patch_specs=ext_options.get('patches', []), extension=True) if ext_patches: self.log.debug('Found patches for extension %s: %s' % (ext_name, ext_patches)) ext_src.update({'patches': ext_patches}) @@ -472,7 +482,7 @@ def obtain_file(self, filename, extension=False, urls=None): common_filepaths = [] if self.robot_path: common_filepaths.extend(self.robot_path) - common_filepaths.extend(get_paths_for("easyconfigs", robot_path=self.robot_path)) + common_filepaths.extend(get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=self.robot_path)) for path in ebpath + common_filepaths + srcpaths: # create list of candidate filepaths @@ -603,6 +613,13 @@ def short_mod_name(self): """ return self.cfg.short_mod_name + @property + def moduleGenerator(self): + """ + Module generator (DEPRECATED, use self.module_generator instead). + """ + self.log.nosupport("self.moduleGenerator is replaced by self.module_generator", '2.0') + # # DIRECTORY UTILITY FUNCTIONS # @@ -743,14 +760,14 @@ def make_devel_module(self, create_in_builddir=False): # these should be all the dependencies and we should load them for key in os.environ: # legacy support - if key.startswith(DEVEL_ENV_VAR_NAME_PREFIX) or key.startswith("SOFTDEVEL"): - if key.startswith("SOFTDEVEL"): - self.log.deprecated("Environment variable SOFTDEVEL* being relied on", "2.0") + if key.startswith(DEVEL_ENV_VAR_NAME_PREFIX): if not key.endswith(convert_name(self.name, upper=True)): path = os.environ[key] if os.path.isfile(path): mod_name = path.rsplit(os.path.sep, 1)[-1] load_txt += mod_gen.load_module(mod_name) + elif key.startswith('SOFTDEVEL'): + self.log.nosupport("Environment variable SOFTDEVEL* being relied on", '2.0') if create_in_builddir: output_dir = self.builddir @@ -806,8 +823,8 @@ def make_module_dep(self): deps = [d for d in deps if d not in excluded_deps] self.log.debug("List of retained dependencies: %s" % deps) - loads = [self.moduleGenerator.load_module(d) for d in deps] - unloads = [self.moduleGenerator.unload_module(d) for d in deps[::-1]] + loads = [self.module_generator.load_module(d) for d in deps] + unloads = [self.module_generator.unload_module(d) for d in deps[::-1]] # Force unloading any other modules if self.cfg['moduleforceunload']: @@ -819,7 +836,7 @@ def make_module_description(self): """ Create the module description. """ - return self.moduleGenerator.get_description() + return self.module_generator.get_description() def make_module_extra(self): """ @@ -832,26 +849,26 @@ def make_module_extra(self): #todo this is only valid for Lua now # This is a bit different in Lua due to string quoting rules in Lua and in Tcl - so $root cannot be used easily. # so we resort to rendering our internal variables and quote them in the set_environment() like all other values. - txt += self.moduleGenerator.set_environment(ROOT_ENV_VAR_NAME_PREFIX + environment_name, self.installdir) - txt += self.moduleGenerator.set_environment(VERSION_ENV_VAR_NAME_PREFIX + environment_name, self.version) + txt += self.module_generator.set_environment(ROOT_ENV_VAR_NAME_PREFIX + environment_name, self.installdir) + txt += self.module_generator.set_environment(VERSION_ENV_VAR_NAME_PREFIX + environment_name, self.version) devel_path = os.path.join(self.installdir, log_path(), ActiveMNS().det_devel_module_filename(self.cfg)) - txt += self.moduleGenerator.set_environment(DEVEL_ENV_VAR_NAME_PREFIX + environment_name, devel_path) + txt += self.module_generator.set_environment(DEVEL_ENV_VAR_NAME_PREFIX + environment_name, devel_path) txt += "\n" for (key, value) in self.cfg['modextravars'].items(): - txt += self.moduleGenerator.set_environment(key, value) + txt += self.module_generator.set_environment(key, value) for (key, value) in self.cfg['modextrapaths'].items(): if isinstance(value, basestring): value = [value] elif not isinstance(value, (tuple, list)): self.log.error("modextrapaths dict value %s (type: %s) is not a list or tuple" % (value, type(value))) - txt += self.moduleGenerator.prepend_paths(key, value) + txt += self.module_generator.prepend_paths(key, value) if self.cfg['modloadmsg']: - txt += self.moduleGenerator.msg_on_load(self.cfg['modloadmsg']) + txt += self.module_generator.msg_on_load(self.cfg['modloadmsg']) if self.cfg['modtclfooter']: - txt += self.moduleGenerator.add_tcl_footer(self.cfg['modtclfooter']) + txt += self.module_generator.add_tcl_footer(self.cfg['modtclfooter']) for (key, value) in self.cfg['modaliases'].items(): - txt += self.moduleGenerator.set_alias(key, value) + txt += self.module_generator.set_alias(key, value) self.log.debug("make_module_extra added this: %s" % txt) @@ -867,7 +884,8 @@ def make_module_extra_extensions(self): # set environment variable that specifies list of extensions if self.exts_all: exts_list = ','.join(['%s-%s' % (ext['name'], ext.get('version', '')) for ext in self.exts_all]) - txt += self.moduleGenerator.set_environment('EBEXTSLIST%s' % self.name.upper(), exts_list) + env_var_name = convert_name(self.name, upper=True) + txt += self.module_generator.set_environment('EBEXTSLIST%s' % env_var_name, exts_list) return txt @@ -903,7 +921,7 @@ def make_module_extend_modpath(self): # module path extensions must exist, otherwise loading this module file will fail for modpath_extension in full_path_modpath_extensions: mkdir(modpath_extension, parents=True) - txt = self.moduleGenerator.use(full_path_modpath_extensions) + txt = self.module_generator.use(full_path_modpath_extensions) else: self.log.debug("Not including module path extensions, as specified.") return txt @@ -916,7 +934,6 @@ def make_module_req(self): if os.path.exists(self.installdir): try: - cwd = os.getcwd() os.chdir(self.installdir) except OSError, err: self.log.error("Failed to change to %s: %s" % (self.installdir, err)) @@ -926,11 +943,11 @@ def make_module_req(self): for path in requirements[key]: paths = glob.glob(path) if paths: - txt += self.moduleGenerator.prepend_paths(key, paths) + txt += self.module_generator.prepend_paths(key, paths) try: - os.chdir(cwd) + os.chdir(self.orig_workdir) except OSError, err: - self.log.error("Failed to change back to %s: %s" % (cwd, err)) + self.log.error("Failed to change back to %s: %s" % (self.orig_workdir, err)) else: txt = "" return txt @@ -1001,7 +1018,7 @@ def clean_up_fake_module(self, fake_mod_data): self.log.warning("Not unloading module, since self.full_mod_name is not set.") # restore original environment - modify_env(os.environ, orig_env) + restore_env(orig_env) def load_dependency_modules(self): """Load dependency modules.""" @@ -1050,14 +1067,13 @@ def skip_extensions(self): 'src': ext.get('source'), } - deprecated_msg = "Providing 'name' and 'version' keys for extensions, should use 'ext_name', 'ext_version'" - self.log.deprecated(deprecated_msg, '2.0') - tmpldict.update({ - 'name': modname, - 'version': ext.get('version'), - }) + try: + cmd = cmdtmpl % tmpldict + except KeyError, err: + msg = "KeyError occured on completing extension filter template: %s; " + msg += "'name'/'version' keys are no longer supported, should use 'ext_name'/'ext_version' instead" + self.log.nosupport(msg, '2.0') - cmd = cmdtmpl % tmpldict if cmdinputtmpl: stdin = cmdinputtmpl % tmpldict (cmdstdouterr, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, regexp=False) @@ -1193,7 +1209,7 @@ def fetch_step(self, skip_checksums=False): # fetch extensions if len(self.cfg['exts_list']) > 0: self.exts = self.fetch_extension_sources() - + # fetch patches if self.cfg['patches']: if isinstance(self.cfg['checksums'], (list, tuple)): @@ -1201,7 +1217,7 @@ def fetch_step(self, skip_checksums=False): patches_checksums = self.cfg['checksums'][len(self.cfg['sources']):] else: patches_checksums = self.cfg['checksums'] - self.fetch_patches(self.cfg['patches'], checksums=patches_checksums) + self.fetch_patches(checksums=patches_checksums) else: self.log.info('no patches provided') @@ -1260,28 +1276,32 @@ def patch_step(self, beginpath=None): for patch in self.patches: self.log.info("Applying patch %s" % patch['name']) - copy = False - # default: patch first source - srcind = 0 - if 'source' in patch: - srcind = patch['source'] - srcpathsuffix = '' - if 'sourcepath' in patch: - srcpathsuffix = patch['sourcepath'] - elif 'copy' in patch: - srcpathsuffix = patch['copy'] - copy = True - - if not beginpath: - beginpath = self.src[srcind]['finalpath'] + # patch source at specified index (first source if not specified) + srcind = patch.get('source', 0) + # if patch level is specified, use that (otherwise let apply_patch derive patch level) + level = patch.get('level', None) + # determine suffix of source path to apply patch in (if any) + srcpathsuffix = patch.get('sourcepath', patch.get('copy', '')) + # determine whether 'patch' file should be copied rather than applied + copy_patch = 'copy' in patch and not 'sourcepath' in patch - src = os.path.abspath("%s/%s" % (beginpath, srcpathsuffix)) + tup = (srcind, level, srcpathsuffix, copy) + self.log.debug("Source index: %s; patch level: %s; source path suffix: %s; copy patch: %s" % tup) - level = None - if 'level' in patch: - level = patch['level'] + if beginpath is None: + try: + beginpath = self.src[srcind]['finalpath'] + self.log.debug("Determine begin path for patch %s: %s" % (patch['name'], beginpath)) + except IndexError, err: + tup = (patch['name'], srcind, self.src, err) + self.log.error("Can't apply patch %s to source at index %s of list %s: %s" % tup) + else: + self.log.debug("Using specified begin path for patch %s: %s" % (patch['name'], beginpath)) + + src = os.path.abspath("%s/%s" % (beginpath, srcpathsuffix)) + self.log.debug("Applying patch %s in path %s" % (patch, src)) - if not apply_patch(patch['path'], src, copy=copy, level=level): + if not apply_patch(patch['path'], src, copy=copy_patch, level=level): self.log.error("Applying patch %s failed" % patch['name']) def prepare_step(self): @@ -1338,7 +1358,7 @@ def extensions_step(self, fetch=False): if fetch: self.exts = self.fetch_extension_sources() - + self.exts_all = self.exts[:] # retain a copy of all extensions, regardless of filtering/skipping if self.skip: @@ -1355,18 +1375,8 @@ def extensions_step(self, fetch=False): self.log.error("ERROR: No default extension class set for %s" % self.name) # obtain name and module path for default extention class - legacy = False if hasattr(exts_defaultclass, '__iter__'): - # LEGACY: module path is explicitely specified - self.log.deprecated("Using specified module path for default class", "2.0") - default_class_modpath = exts_defaultclass[0] - default_class = exts_defaultclass[1] - derived_mod_path = get_module_path(default_class, generic=True) - if not default_class_modpath == derived_mod_path: - msg = "Specified module path for default class %s " % default_class_modpath - msg += "doesn't match derived path %s" % derived_mod_path - self.log.warning(msg) - legacy = True + self.log.nosupport("Module path for default class is explicitly defined", '2.0') elif isinstance(exts_defaultclass, basestring): # proper way: derive module path from specified class name @@ -1380,41 +1390,23 @@ def extensions_step(self, fetch=False): for ext in self.exts: self.log.debug("Starting extension %s" % ext['name']) - # always go back to build dir to avoid running stuff from a dir that no longer exists - os.chdir(self.builddir) - - inst = None + # always go back to original work dir to avoid running stuff from a dir that no longer exists + os.chdir(self.orig_workdir) - # try instantiating extension-specific class - class_name = encode_class_name(ext['name']) # use the same encoding as get_class + cls, inst = None, None + class_name = encode_class_name(ext['name']) mod_path = get_module_path(class_name) - if not os.path.exists("%s.py" % mod_path): - self.log.deprecated("Determine module path based on software name", "2.0") - mod_path = get_module_path(ext['name'], decode=False) + # try instantiating extension-specific class try: - cls = get_class_for(mod_path, class_name) - inst = cls(self, ext) - except (ImportError, NameError), err: - self.log.debug("Failed to use class %s from %s for extension %s: %s" % (class_name, - mod_path, - ext['name'], - err)) - - # LEGACY: try and use default module path for getting extension class instance - if inst is None and legacy: - try: - msg = "Failed to use derived module path for %s, " % class_name - msg += "considering specified module path as (legacy) fallback." - self.log.debug(msg) - mod_path = default_class_modpath - cls = get_class_for(mod_path, class_name) + # no error when importing class fails, in case we run into an existing easyblock + # with a similar name (e.g., Perl Extension 'GO' vs 'Go' for which 'EB_Go' is available) + cls = get_easyblock_class(None, name=ext['name'], default_fallback=False, error_on_failed_import=False) + self.log.debug("Obtained class %s for extension %s" % (cls, ext['name'])) + if cls is not None: inst = cls(self, ext) - except (ImportError, NameError), err: - self.log.debug("Failed to use class %s from %s for extension %s: %s" % (class_name, - mod_path, - ext['name'], - err)) + except (ImportError, NameError), err: + self.log.debug("Failed to use extension-specific class for extension %s: %s" % (ext['name'], err)) # alternative attempt: use class specified in class map (if any) if inst is None and ext['name'] in exts_classmap: @@ -1425,14 +1417,10 @@ def extensions_step(self, fetch=False): cls = get_class_for(mod_path, class_name) inst = cls(self, ext) except (ImportError, NameError), err: - self.log.error("Failed to load specified class %s for extension %s: %s" % (class_name, - ext['name'], - err)) + self.log.error("Failed to load specified class %s for extension %s: %s" % (class_name, ext['name'], err)) # fallback attempt: use default class - if not inst is None: - self.log.debug("Installing extension %s with class %s (from %s)" % (ext['name'], class_name, mod_path)) - else: + if inst is None: try: cls = get_class_for(default_class_modpath, default_class) self.log.debug("Obtained class %s for installing extension %s" % (cls, ext['name'])) @@ -1443,6 +1431,8 @@ def extensions_step(self, fetch=False): msg = "Also failed to use default class %s from %s for extension %s: %s, giving up" % \ (default_class, default_class_modpath, ext['name'], err) self.log.error(msg) + else: + self.log.debug("Installing extension %s with class %s (from %s)" % (ext['name'], class_name, mod_path)) # real work inst.prerun() @@ -1475,7 +1465,7 @@ def post_install_step(self): for cmd in self.cfg['postinstallcmds']: if not isinstance(cmd, basestring): self.log.error("Invalid element in 'postinstallcmds', not a string: %s" % cmd) - run_cmd(cmd, simple=True, log_ok=False, log_all=False) + run_cmd(cmd, simple=True, log_ok=True, log_all=True) if self.group is not None: # remove permissions for others, and set group ID @@ -1626,7 +1616,7 @@ def cleanup_step(self): """ if not self.build_in_installdir and build_option('cleanup_builddir'): try: - os.chdir(build_path()) # make sure we're out of the dir we're removing + os.chdir(self.orig_workdir) # make sure we're out of the dir we're removing self.log.info("Cleaning up builddir %s (in %s)" % (self.builddir, os.getcwd())) @@ -1651,8 +1641,8 @@ def make_module_step(self, fake=False): """ Generate a module file. """ - self.moduleGenerator.set_fake(fake) - modpath = self.moduleGenerator.prepare() + self.module_generator.set_fake(fake) + modpath = self.module_generator.prepare() txt = '' txt += self.make_module_description() @@ -1662,12 +1652,12 @@ def make_module_step(self, fake=False): txt += self.make_module_extra() txt += self.make_module_footer() - write_file(self.moduleGenerator.filename+".lua", txt) + write_file(self.module_generator.filename+".lua", txt) - self.log.info("Module file %s written" % self.moduleGenerator.filename) + self.log.info("Module file %s written" % self.module_generator.filename) self.modules_tool.update() - self.moduleGenerator.create_symlinks() + self.module_generator.create_symlinks() if not fake: self.make_devel_module() @@ -1679,8 +1669,7 @@ def test_cases_step(self): Run provided test cases. """ for test in self.cfg['tests']: - # Current working dir no longer exists - os.chdir(self.installdir) + os.chdir(self.orig_workdir) if os.path.isabs(test): path = test else: @@ -1855,7 +1844,7 @@ def build_and_install_one(module, orig_environ): # restore original environment _log.info("Resetting environment") filetools.errors_found_in_log = 0 - modify_env(os.environ, orig_environ) + restore_env(orig_environ) cwd = os.getcwd() @@ -1950,10 +1939,14 @@ def build_and_install_one(module, orig_environ): try: newspec = os.path.join(new_log_dir, "%s-%s.eb" % (app.name, det_full_ec_version(app.cfg))) - shutil.copy(spec, newspec) - _log.debug("Copied easyconfig file %s to %s" % (spec, newspec)) + # only copy if the files are not the same file already (yes, it happens) + if os.path.exists(newspec) and os.path.samefile(spec, newspec): + _log.debug("Not copying easyconfig file %s to %s since files are identical" % (spec, newspec)) + else: + shutil.copy(spec, newspec) + _log.debug("Copied easyconfig file %s to %s" % (spec, newspec)) except (IOError, OSError), err: - print_error("Failed to move easyconfig %s to log dir %s: %s" % (spec, new_log_dir, err)) + print_error("Failed to copy easyconfig %s to %s: %s" % (spec, newspec, err)) # build failed else: @@ -1986,6 +1979,7 @@ def build_and_install_one(module, orig_environ): return (success, application_log, errormsg) + def get_easyblock_instance(easyconfig): """ Get an instance for this easyconfig @@ -2058,7 +2052,7 @@ def perform_step(step, obj, method, logfile): # start with a clean slate os.chdir(base_dir) - modify_env(os.environ, base_env) + restore_env(base_env) steps = EasyBlock.get_steps(iteration_count=app.det_iter_cnt()) diff --git a/easybuild/framework/easyconfig/__init__.py b/easybuild/framework/easyconfig/__init__.py index b8f643bab3..24e58f0208 100644 --- a/easybuild/framework/easyconfig/__init__.py +++ b/easybuild/framework/easyconfig/__init__.py @@ -31,5 +31,8 @@ from easybuild.framework.easyconfig.default import ALL_CATEGORIES globals().update(ALL_CATEGORIES) +# subdirectory (of 'easybuild' dir) in which easyconfig files are located in a package +EASYCONFIGS_PKG_SUBDIR = 'easyconfigs' + # is used in some tools from easybuild.framework.easyconfig.easyconfig import EasyConfig diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 56b8143f7e..0d33bd8388 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -35,37 +35,28 @@ """ from vsc.utils import fancylogger -from easybuild.tools.ordereddict import OrderedDict _log = fancylogger.getLogger('easyconfig.default', fname=False) # we use a tuple here so we can sort them based on the numbers -HIDDEN = "HIDDEN" -MANDATORY = "MANDATORY" -CUSTOM = "CUSTOM" -TOOLCHAIN = "TOOLCHAIN" -BUILD = "BUILD" -FILEMANAGEMENT = "FILEMANAGEMENT" -DEPENDENCIES = "DEPENDENCIES" -LICENSE = "LICENSE" -EXTENSIONS = "EXTENSIONS" -MODULES = "MODULES" -OTHER = "OTHER" - ALL_CATEGORIES = { - HIDDEN: (-1, 'hidden'), - MANDATORY: (0, 'mandatory'), - CUSTOM: (1, 'easyblock-specific'), - TOOLCHAIN: (2, 'toolchain'), - BUILD: (3, 'build'), - FILEMANAGEMENT: (4, 'file-management'), - DEPENDENCIES: (5, 'dependencies'), - LICENSE: (6, 'license'), - EXTENSIONS: (7, 'extensions'), - MODULES: (8, 'modules'), - OTHER: (9, 'other'), + 'HIDDEN': (-1, 'hidden'), + 'MANDATORY': (0, 'mandatory'), + 'CUSTOM': (1, 'easyblock-specific'), + 'TOOLCHAIN': (2, 'toolchain'), + 'BUILD': (3, 'build'), + 'FILEMANAGEMENT': (4, 'file-management'), + 'DEPENDENCIES': (5, 'dependencies'), + 'LICENSE': (6, 'license'), + 'EXTENSIONS': (7, 'extensions'), + 'MODULES': (8, 'modules'), + 'OTHER': (9, 'other'), } +# define constants so they can be used below +# avoid that pylint complains about unknown variables in this file +# pylint: disable=E0602 +globals().update(ALL_CATEGORIES) # List of tuples. Each tuple has the following format (key, [default, help text, category]) DEFAULT_CONFIG = { @@ -92,7 +83,8 @@ 'buildopts': ['', 'Extra options passed to make step (default already has -j X)', BUILD], 'checksums': [[], "Checksums for sources and patches", BUILD], 'configopts': ['', 'Extra options passed to configure (default already has --prefix)', BUILD], - 'easyblock': ['ConfigureMake', "EasyBlock to use for building", BUILD], + 'easyblock': [None, "EasyBlock to use for building; if set to None, an easyblock is selected " + "based on the software name", BUILD], 'easybuild_version': [None, "EasyBuild-version this spec-file was written for", BUILD], 'installopts': ['', 'Extra options for installation', BUILD], 'maxparallel': [None, 'Max degree of parallelism', BUILD], @@ -185,28 +177,6 @@ def sorted_categories(): return categories -def convert_to_help(opts, has_default=False): - """ - Converts the given list to a mapping of category -> [(name, help)] (OrderedDict) - @param: has_default, if False, add the DEFAULT_CONFIG list - """ - mapping = OrderedDict() - if isinstance(opts, dict): - opts = opts.items() - if not has_default: - defs = [(k, [def_val, descr, ALL_CATEGORIES[cat]]) for k, (def_val, descr, cat) in DEFAULT_CONFIG.items()] - opts = defs + opts - - # sort opts - opts.sort() - - for cat in sorted_categories(): - mapping[cat[1]] = [(opt[0], "%s (default: %s)" % (opt[1][1], opt[1][0])) - for opt in opts if opt[1][2] == cat] - - return mapping - - def get_easyconfig_parameter_default(param): """Get default value for given easyconfig parameter.""" if param not in DEFAULT_CONFIG: diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 3fea3e6c3e..9db79d881d 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -40,7 +40,7 @@ import os import re from vsc.utils import fancylogger -from vsc.utils.missing import any, nub +from vsc.utils.missing import get_class_for, nub from vsc.utils.patterns import Singleton import easybuild.tools.environment as env @@ -56,7 +56,7 @@ from easybuild.tools.toolchain.utilities import get_toolchain from easybuild.tools.utilities import remove_unwanted_chars from easybuild.framework.easyconfig import MANDATORY -from easybuild.framework.easyconfig.default import DEFAULT_CONFIG, ALL_CATEGORIES, get_easyconfig_parameter_default +from easybuild.framework.easyconfig.default import DEFAULT_CONFIG, get_easyconfig_parameter_default from easybuild.framework.easyconfig.format.convert import Dependency from easybuild.framework.easyconfig.format.one import retrieve_blocks_in_spec from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT, License @@ -66,57 +66,39 @@ _log = fancylogger.getLogger('easyconfig.easyconfig', fname=False) - # add license here to make it really MANDATORY (remove comment in default) -_log.deprecated('Mandatory license not enforced', '2.0') MANDATORY_PARAMS = ['name', 'version', 'homepage', 'description', 'toolchain'] # set of configure/build/install options that can be provided as lists for an iterated build ITERATE_OPTIONS = ['preconfigopts', 'configopts', 'prebuildopts', 'buildopts', 'preinstallopts', 'installopts'] -# map of deprecated easyconfig parameters, and their replacements -DEPRECATED_OPTIONS = { - 'license': ('software_license', '2.0'), - 'makeopts': ('buildopts', '2.0'), - 'premakeopts': ('prebuildopts', '2.0'), +# deprecated easyconfig parameters, and their replacements +DEPRECATED_PARAMETERS = { + # : (, ), +} + +# replaced easyconfig parameters, and their replacements +REPLACED_PARAMETERS = { + 'license': 'software_license', + 'makeopts': 'buildopts', + 'premakeopts': 'prebuildopts', } _easyconfig_files_cache = {} _easyconfigs_cache = {} -def handle_deprecated_easyconfig_parameter(ec_method): - """Decorator to handle deprecated easyconfig parameters.""" +def handle_deprecated_or_replaced_easyconfig_parameters(ec_method): + """Decorator to handle deprecated/replaced easyconfig parameters.""" def new_ec_method(self, key, *args, **kwargs): - """Map deprecated easyconfig parameters to the new correct parameter.""" - # map name of deprecated easyconfig parameter to new name - if key in DEPRECATED_OPTIONS: + """Check whether any replace easyconfig parameters are still used""" + # map deprecated parameters to their replacements, issue deprecation warning(/error) + if key in DEPRECATED_PARAMETERS: depr_key = key - key, ver = DEPRECATED_OPTIONS[depr_key] + key, ver = DEPRECATED_PARAMETERS[depr_key] _log.deprecated("Easyconfig parameter '%s' is deprecated, use '%s' instead." % (depr_key, key), ver) - - # make sure that value for software_license has correct type, convert if needed - if key == 'software_license': - # key 'license' will already be mapped to 'software_license' above - lic = self._config['software_license'] - if not isinstance(lic, License): - self.log.deprecated('Type for software_license must to be instance of License (sub)class', '2.0') - lic_type = type(lic) - - class LicenseLegacy(License, lic_type): - """A special License class to deal with legacy license paramters""" - DESCRICPTION = ("Internal-only, legacy closed license class to deprecate license parameter." - " (DO NOT USE).") - HIDDEN = False - - def __init__(self, *args): - if len(args) > 0: - lic_type.__init__(self, args[0]) - License.__init__(self) - lic = LicenseLegacy(lic) - EASYCONFIG_LICENSES_DICT[lic.name] = lic - self._config['software_license'] = lic - + if key in REPLACED_PARAMETERS: + _log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0') return ec_method(self, key, *args, **kwargs) return new_ec_method @@ -149,10 +131,7 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi if self.valid_module_classes is not None: self.log.info("Obtained list of valid module classes: %s" % self.valid_module_classes) - # replace the category name with the category - self._config = {} - for k, [def_val, descr, cat] in copy.deepcopy(DEFAULT_CONFIG).items(): - self._config[k] = [def_val, descr, ALL_CATEGORIES[cat]] + self._config = copy.deepcopy(DEFAULT_CONFIG) if extra_options is None: name = fetch_parameter_from_easyconfig_file(path, 'name') @@ -163,27 +142,13 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi self.extra_options = extra_options if not isinstance(self.extra_options, dict): - if isinstance(self.extra_options, (list, tuple,)): - typ = type(self.extra_options) - self.log.deprecated("Specified extra_options should be of type 'dict', found type '%s'" % typ, '2.0') - tup = (self.extra_options, type(self.extra_options)) - self.log.debug("Converting extra_options value '%s' of type '%s' to a dict" % tup) - self.extra_options = dict(self.extra_options) - else: - tup = (type(self.extra_options), self.extra_options) - self.log.error("extra_options parameter passed is of incorrect type: %s ('%s')" % tup) - - # map deprecated params to new names if they occur in extra_options - for key, val in self.extra_options.items(): - if key in DEPRECATED_OPTIONS: - new_key, depr_ver = DEPRECATED_OPTIONS[key] - self.log.deprecated("Found deprecated key '%s', should use '%s' instead." % (key, new_key), depr_ver) - self.extra_options[new_key] = self.extra_options[key] - self.log.debug("Set '%s' with value of deprecated '%s': %s" % (new_key, key, self.extra_options[key])) - del self.extra_options[key] + tup = (type(self.extra_options), self.extra_options) + self.log.nosupport("extra_options return value should be of type 'dict', found '%s': %s" % tup, '2.0') + self._config.update(self.extra_options) self.path = path + self.mandatory = MANDATORY_PARAMS[:] # extend mandatory keys @@ -276,7 +241,7 @@ def parse(self): # provide suggestions for typos possible_typos = [(key, difflib.get_close_matches(key.lower(), self._config.keys(), 1, 0.85)) - for key in local_vars if key not in self._config] + for key in local_vars if key not in self] typos = [(key, guesses[0]) for (key, guesses) in possible_typos if len(guesses) == 1] if typos: @@ -287,7 +252,7 @@ def parse(self): for key in ['toolchain'] + local_vars.keys(): # validations are skipped, just set in the config # do not store variables we don't need - if key in self._config.keys() + DEPRECATED_OPTIONS.keys(): + if key in self._config.keys(): if key in ['builddependencies', 'dependencies']: self[key] = [self._parse_dependency(dep) for dep in local_vars[key]] elif key in ['hiddendependencies']: @@ -296,6 +261,8 @@ def parse(self): self[key] = local_vars[key] tup = (key, self[key], type(self[key])) self.log.info("setting config option %s: value %s (type: %s)" % tup) + elif key in REPLACED_PARAMETERS: + _log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0') else: self.log.debug("Ignoring unknown config option %s (value: %s)" % (key, local_vars[key])) @@ -339,14 +306,16 @@ def validate(self, check_osdeps=True): self.log.info("Checking licenses") self.validate_license() + self.log.info("Checking whether list of hidden dependencies is a subset of list of dependencies") + self.validate_hiddendeps() + def validate_license(self): """Validate the license""" lic = self._config['software_license'][0] if lic is None: - self.log.deprecated('Mandatory license not enforced', '2.0') # when mandatory, remove this possibility if 'software_license' in self.mandatory: - self.log.error('License is mandatory') + self.log.error("License is mandatory, but 'software_license' is undefined") elif not isinstance(lic, License): self.log.error('License %s has to be a License subclass instance, found classname %s.' % (lic, lic.__class__.__name__)) @@ -407,6 +376,30 @@ def validate_iterate_opts_lists(self): return True + def validate_hiddendeps(self): + """ + Validate that list of hidden dependencies is a subset of the list of dependencies. + The list of dependencies is adjusted to only include non-hidden dependencies. + """ + dep_mod_names = [dep['full_mod_name'] for dep in self['dependencies']] + + faulty_deps = [] + for hidden_dep in self['hiddendependencies']: + # check whether hidden dep is a listed dep using *visible* module name, not hidden one + visible_mod_name = ActiveMNS().det_full_module_name(hidden_dep, force_visible=True) + if visible_mod_name in dep_mod_names: + self['dependencies'] = [d for d in self['dependencies'] if d['full_mod_name'] != visible_mod_name] + self.log.debug("Removed dependency matching hidden dependency %s" % hidden_dep) + else: + # hidden dependencies must also be included in list of dependencies; + # this is done to try and make easyconfigs portable w.r.t. site-specific policies with minimal effort, + # i.e. by simply removing the 'hiddendependencies' specification + faulty_deps.append(visible_mod_name) + + if faulty_deps: + tup = (faulty_deps, dep_mod_names) + self.log.error("Hidden dependencies with visible module names %s not in list of dependencies: %s" % tup) + def dependencies(self): """ Returns an array of parsed dependencies (after filtering, if requested) @@ -636,33 +629,43 @@ def _generate_template_values(self, ignore=None, skip_lower=True): if v is None: del self.template_values[k] - @handle_deprecated_easyconfig_parameter + @handle_deprecated_or_replaced_easyconfig_parameters + def __contains__(self, key): + """Check whether easyconfig parameter is defined""" + return key in self._config + + @handle_deprecated_or_replaced_easyconfig_parameters def __getitem__(self, key): - """ - will return the value without the help text - """ - value = self._config[key][0] + """Return value of specified easyconfig parameter (without help text, etc.)""" + value = None + if key in self._config: + value = self._config[key][0] + else: + self.log.error("Use of unknown easyconfig parameter '%s' when getting parameter value" % key) + if self.enable_templating: if self.template_values is None or len(self.template_values) == 0: self.generate_template_values() - return resolve_template(value, self.template_values) - else: - return value + value = resolve_template(value, self.template_values) - @handle_deprecated_easyconfig_parameter + return value + + @handle_deprecated_or_replaced_easyconfig_parameters def __setitem__(self, key, value): - """ - sets the value of key in config. - help text is untouched - """ - self._config[key][0] = value + """Set value of specified easyconfig parameter (help text & co is left untouched)""" + if key in self._config: + self._config[key][0] = value + else: + tup = (key, value) + self.log.error("Use of unknown easyconfig parameter '%s' when setting parameter value to '%s'" % tup) + @handle_deprecated_or_replaced_easyconfig_parameters def get(self, key, default=None): """ Gets the value of a key in the config, with 'default' as fallback. """ - if key in self._config: - return self.__getitem__(key) + if key in self: + return self[key] else: return default @@ -684,21 +687,14 @@ def asdict(self): def det_installversion(version, toolchain_name, toolchain_version, prefix, suffix): """Deprecated 'det_installversion' function, to determine exact install version, based on supplied parameters.""" old_fn = 'framework.easyconfig.easyconfig.det_installversion' - _log.deprecated('Use module_generator.det_full_ec_version instead of %s' % old_fn, '2.0') - cfg = { - 'version': version, - 'toolchain': {'name': toolchain_name, 'version': toolchain_version}, - 'versionprefix': prefix, - 'versionsuffix': suffix, - } - return det_full_ec_version(cfg) + _log.nosupport('Use det_full_ec_version from easybuild.tools.module_generator instead of %s' % old_fn, '2.0') def fetch_parameter_from_easyconfig_file(path, param): """Fetch parameter specification from given easyconfig file.""" # check whether easyblock is specified in easyconfig file # note: we can't rely on value for 'easyblock' in parsed easyconfig, it may be the default value - reg = re.compile(r"^\s*%s\s*=\s*(?P\S.*)\s*$" % param, re.M) + reg = re.compile(r"^\s*%s\s*=\s*(?P\S.*?)\s*$" % param, re.M) txt = read_file(path) res = reg.search(txt) if res: @@ -707,31 +703,11 @@ def fetch_parameter_from_easyconfig_file(path, param): return None -def get_class_for(modulepath, class_name): - """ - Get class for a given class name and easyblock module path. - """ - # try to import specified module path, reraise ImportError if it occurs - try: - m = __import__(modulepath, globals(), locals(), ['']) - except ImportError, err: - raise ImportError(err) - # try to import specified class name from specified module path, throw ImportError if this fails - try: - c = getattr(m, class_name) - except AttributeError, err: - raise ImportError("Failed to import %s from %s: %s" % (class_name, modulepath, err)) - return c - - -def get_easyblock_class(easyblock, name=None): +def get_easyblock_class(easyblock, name=None, default_fallback=True, error_on_failed_import=True): """ Get class for a particular easyblock (or use default) """ - - def_class = get_easyconfig_parameter_default('easyblock') - def_mod_path = get_module_path(def_class, generic=True) - + cls = None try: if easyblock: # something was specified, lets parse it @@ -761,9 +737,22 @@ def get_easyblock_class(easyblock, name=None): class_name = encode_class_name(name) # modulepath will be the namespace + encoded modulename (from the classname) modulepath = get_module_path(class_name) - if not os.path.exists("%s.py" % modulepath): - _log.deprecated("Determine module path based on software name", "2.0") - modulepath = get_module_path(name, decode=False) + modulepath_imported = False + try: + __import__(modulepath, globals(), locals(), ['']) + modulepath_imported = True + except ImportError, err: + _log.debug("Failed to import module '%s': %s" % (modulepath, err)) + + # check if determining module path based on software name would have resulted in a different module path + if modulepath_imported: + _log.debug("Module path '%s' found" % modulepath) + else: + _log.debug("No module path '%s' found" % modulepath) + modulepath_bis = get_module_path(name, decode=False) + _log.debug("Module path determined based on software name: %s" % modulepath_bis) + if modulepath_bis != modulepath: + _log.nosupport("Determining module path based on software name", '2.0') # try and find easyblock try: @@ -771,23 +760,36 @@ def get_easyblock_class(easyblock, name=None): cls = get_class_for(modulepath, class_name) _log.info("Successfully obtained %s class instance from %s" % (class_name, modulepath)) except ImportError, err: - # when an ImportError occurs, make sure that it's caused by not finding the easyblock module, # and not because of a broken import statement in the easyblock module error_re = re.compile(r"No module named %s" % modulepath.replace("easybuild.easyblocks.", '')) _log.debug("error regexp: %s" % error_re.pattern) if error_re.match(str(err)): - # no easyblock could be found, so fall back to default class. - _log.warning("Failed to import easyblock for %s, falling back to default class %s: error: %s" % \ - (class_name, (def_mod_path, def_class), err)) - cls = get_class_for(def_mod_path, def_class) + if default_fallback: + # no easyblock could be found, so fall back to ConfigureMake (NO LONGER SUPPORTED) + legacy_fallback_easyblock = 'ConfigureMake' + def_mod_path = get_module_path(legacy_fallback_easyblock, generic=True) + depr_msg = "Fallback to default easyblock %s (from %s)" % (legacy_fallback_easyblock, def_mod_path) + depr_msg += "; use \"easyblock = '%s'\" in easyconfig file?" % legacy_fallback_easyblock + _log.nosupport(depr_msg, '2.0') else: - _log.error("Failed to import easyblock for %s because of module issue: %s" % (class_name, err)) + if error_on_failed_import: + _log.error("Failed to import easyblock for %s because of module issue: %s" % (class_name, err)) + else: + _log.debug("Failed to import easyblock for %s, but ignoring it: %s" % (class_name, err)) + + if cls is not None: + tup = (cls.__name__, easyblock, name) + _log.info("Successfully obtained class '%s' for easyblock '%s' (software name '%s')" % tup) + else: + tup = (easyblock, name, default_fallback) + _log.debug("No class found for easyblock '%s' (software name '%s', default fallback: %s" % tup) - tup = (cls.__name__, easyblock, name) - _log.info("Successfully obtained class '%s' for easyblock '%s' (software name '%s')" % tup) return cls + except EasyBuildError, err: + # simply reraise rather than wrapping it into another error + raise err except Exception, err: _log.error("Failed to obtain class for %s easyblock (not available?): %s" % (easyblock, err)) diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index 4003f5682d..7fb06cc2f9 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -32,7 +32,7 @@ import copy import re from vsc.utils import fancylogger -from vsc.utils.missing import get_subclasses, any +from vsc.utils.missing import get_subclasses from easybuild.framework.easyconfig.format.version import EasyVersion, OrderedVersionOperators from easybuild.framework.easyconfig.format.version import ToolchainVersionOperator, VersionOperator diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py index 995931f1e1..7728792947 100644 --- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py +++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py @@ -75,7 +75,6 @@ def build_easyconfig_constants_dict(): def build_easyconfig_variables_dict(): """Make a dictionary with all variables that can be used""" - _log.deprecated("Magic 'global' easyconfigs variables like shared_lib_ext should no longer be used", '2.0') vars_dict = { "shared_lib_ext": get_shared_lib_ext(), } @@ -178,6 +177,11 @@ def parse_pyheader(self, pyheader): self.log.debug("pyheader initial local_vars %s" % local_vars) self.log.debug("pyheader text being exec'ed: %s" % pyheader) + # check for use of deprecated magic easyconfigs variables + for magic_var in build_easyconfig_variables_dict(): + if re.search(magic_var, pyheader, re.M): + _log.nosupport("Magic 'global' easyconfigs variable %s should no longer be used" % magic_var, '2.0') + try: exec(pyheader, global_vars, local_vars) except SyntaxError, err: diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 42ec1038f8..57b973aca2 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -28,7 +28,7 @@ be used within an Easyconfig file. @author: Stijn De Weirdt (Ghent University) -@author: Fotis Georgatos (University of Luxembourg) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ from vsc.utils import fancylogger diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 6a7c48a062..da3af771ac 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -33,7 +33,8 @@ @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) @author: Toon Willems (Ghent University) -@author: Fotis Georgatos (University of Luxembourg) +@author: Fotis Georgatos (Uni.Lu, NTUA) +@author: Ward Poelmans (Ghent University) """ import os @@ -65,21 +66,23 @@ except ImportError, err: graph_errors.append("Failed to import graphviz: try yum install graphviz-python, or apt-get install python-pygraphviz") +from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR from easybuild.framework.easyconfig.easyconfig import ActiveMNS -from easybuild.framework.easyconfig.easyconfig import process_easyconfig, robot_find_easyconfig -from easybuild.tools.build_log import EasyBuildError, print_msg +from easybuild.framework.easyconfig.easyconfig import process_easyconfig +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option -from easybuild.tools.filetools import det_common_path_prefix, run_cmd, write_file -from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS -from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version, det_hidden_modname +from easybuild.tools.filetools import find_easyconfigs, search_file, write_file +from easybuild.tools.github import fetch_easyconfigs_from_pr from easybuild.tools.modules import modules_tool from easybuild.tools.ordereddict import OrderedDict +from easybuild.tools.run import run_cmd from easybuild.tools.utilities import quote_str + _log = fancylogger.getLogger('easyconfig.tools', fname=False) -def skip_available(easyconfigs, testing=False): +def skip_available(easyconfigs): """Skip building easyconfigs for existing modules.""" modtool = modules_tool() module_names = [ec['full_mod_name'] for ec in easyconfigs] @@ -87,28 +90,32 @@ def skip_available(easyconfigs, testing=False): retained_easyconfigs = [] for ec, mod_name, mod_exists in zip(easyconfigs, module_names, modules_exist): if mod_exists: - msg = "%s is already installed (module found), skipping" % mod_name - print_msg(msg, log=_log, silent=testing) - _log.info(msg) + _log.info("%s is already installed (module found), skipping" % mod_name) else: _log.debug("%s is not installed yet, so retaining it" % mod_name) retained_easyconfigs.append(ec) return retained_easyconfigs -def find_resolved_modules(unprocessed, avail_modules): +def find_resolved_modules(unprocessed, avail_modules, retain_all_deps=False): """ Find easyconfigs in 1st argument which can be fully resolved using modules specified in 2nd argument """ ordered_ecs = [] new_avail_modules = avail_modules[:] new_unprocessed = [] + modtool = modules_tool() for ec in unprocessed: new_ec = ec.copy() deps = [] for dep in new_ec['dependencies']: - if not ActiveMNS().det_full_module_name(dep) in new_avail_modules: + full_mod_name = ActiveMNS().det_full_module_name(dep) + dep_resolved = full_mod_name in new_avail_modules + if not retain_all_deps: + # hidden modules need special care, since they may not be included in list of available modules + dep_resolved |= dep['hidden'] and modtool.exist([full_mod_name])[0] + if not dep_resolved: deps.append(dep) new_ec['dependencies'] = deps @@ -123,174 +130,6 @@ def find_resolved_modules(unprocessed, avail_modules): return ordered_ecs, new_unprocessed, new_avail_modules -def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False): - """ - Work through the list of easyconfigs to determine an optimal order - @param unprocessed: list of easyconfigs - @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) - """ - - robot = build_option('robot_path') - - retain_all_deps = build_option('retain_all_deps') or retain_all_deps - if retain_all_deps: - # assume that no modules are available when forced, to retain all dependencies - avail_modules = [] - _log.info("Forcing all dependencies to be retained.") - else: - # Get a list of all available modules (format: [(name, installversion), ...]) - avail_modules = modules_tool().available() - - if len(avail_modules) == 0: - _log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH')) - - ordered_ecs = [] - # all available modules can be used for resolving dependencies except those that will be installed - being_installed = [p['full_mod_name'] for p in unprocessed] - avail_modules = [m for m in avail_modules if not m in being_installed] - - _log.debug('unprocessed before resolving deps: %s' % unprocessed) - - # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though) - irresolvable = [] - loopcnt = 0 - maxloopcnt = 10000 - while unprocessed: - # make sure this stops, we really don't want to get stuck in an infinite loop - loopcnt += 1 - if loopcnt > maxloopcnt: - tup = (maxloopcnt, unprocessed, irresolvable) - msg = "Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)" % tup - _log.error(msg) - - # first try resolving dependencies without using external dependencies - last_processed_count = -1 - while len(avail_modules) > last_processed_count: - last_processed_count = len(avail_modules) - more_ecs, unprocessed, avail_modules = find_resolved_modules(unprocessed, avail_modules) - for ec in more_ecs: - if not ec['full_mod_name'] in [x['full_mod_name'] for x in ordered_ecs]: - ordered_ecs.append(ec) - - # robot: look for existing dependencies, add them - if robot and unprocessed: - - # rely on EasyBuild module naming scheme when resolving dependencies, since we know that will - # generate sensible module names that include the necessary information for the resolution to work - # (name, version, toolchain, versionsuffix) - being_installed = [EasyBuildMNS().det_full_module_name(p['ec']) for p in unprocessed] - - additional = [] - for i, entry in enumerate(unprocessed): - # do not choose an entry that is being installed in the current run - # if they depend, you probably want to rebuild them using the new dependency - deps = entry['dependencies'] - candidates = [d for d in deps if not EasyBuildMNS().det_full_module_name(d) in being_installed] - if len(candidates) > 0: - cand_dep = candidates[0] - # find easyconfig, might not find any - _log.debug("Looking for easyconfig for %s" % str(cand_dep)) - # note: robot_find_easyconfig may return None - path = robot_find_easyconfig(cand_dep['name'], det_full_ec_version(cand_dep)) - - if path is None: - # no easyconfig found for dependency, add to list of irresolvable dependencies - if cand_dep not in irresolvable: - _log.debug("Irresolvable dependency found: %s" % cand_dep) - irresolvable.append(cand_dep) - # remove irresolvable dependency from list of dependencies so we can continue - entry['dependencies'].remove(cand_dep) - else: - _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) - # build specs should not be passed down to resolved dependencies, - # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself - hidden = cand_dep.get('hidden', False) - processed_ecs = process_easyconfig(path, validate=not retain_all_deps, hidden=hidden) - - # ensure that selected easyconfig provides required dependency - mods = [spec['ec'].full_mod_name for spec in processed_ecs] - dep_mod_name = ActiveMNS().det_full_module_name(cand_dep) - if not dep_mod_name in mods: - tup = (path, dep_mod_name, mods) - _log.error("easyconfig file %s does not contain module %s (mods: %s)" % tup) - - for ec in processed_ecs: - if not ec in unprocessed + additional: - additional.append(ec) - _log.debug("Added %s as dependency of %s" % (ec, entry)) - else: - mod_name = EasyBuildMNS().det_full_module_name(entry['ec']) - _log.debug("No more candidate dependencies to resolve for %s" % mod_name) - - # add additional (new) easyconfigs to list of stuff to process - unprocessed.extend(additional) - - elif not robot: - # no use in continuing if robot is not enabled, dependencies won't be resolved anyway - irresolvable = [dep for x in unprocessed for dep in x['dependencies']] - break - - if irresolvable: - _log.warning("Irresolvable dependencies (details): %s" % irresolvable) - irresolvable_mods_eb = [EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable] - _log.warning("Irresolvable dependencies (EasyBuild module names): %s" % ', '.join(irresolvable_mods_eb)) - irresolvable_mods = [ActiveMNS().det_full_module_name(dep) for dep in irresolvable] - _log.error('Irresolvable dependencies encountered: %s' % ', '.join(irresolvable_mods)) - - _log.info("Dependency resolution complete, building as follows:\n%s" % ordered_ecs) - return ordered_ecs - - -def print_dry_run(easyconfigs, short=False, build_specs=None): - """ - Print dry run information - @param easyconfigs: list of easyconfig files - @param short: print short output (use a variable for the common prefix) - @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) - """ - lines = [] - if build_option('robot_path') is None: - lines.append("Dry run: printing build status of easyconfigs") - all_specs = easyconfigs - else: - lines.append("Dry run: printing build status of easyconfigs and dependencies") - all_specs = resolve_dependencies(easyconfigs, build_specs=build_specs, retain_all_deps=True) - - unbuilt_specs = skip_available(all_specs, testing=True) - dry_run_fmt = " * [%1s] %s (module: %s)" # markdown compatible (list of items with checkboxes in front) - - listed_ec_paths = [spec['spec'] for spec in easyconfigs] - - var_name = 'CFGS' - common_prefix = det_common_path_prefix([spec['spec'] for spec in all_specs]) - # only allow short if common prefix is long enough - short = short and common_prefix is not None and len(common_prefix) > len(var_name) * 2 - for spec in all_specs: - if spec in unbuilt_specs: - ans = ' ' - elif build_option('force') and spec['spec'] in listed_ec_paths: - ans = 'F' - else: - ans = 'x' - - if spec['ec'].short_mod_name != spec['ec'].full_mod_name: - mod = "%s | %s" % (spec['ec'].mod_subdir, spec['ec'].short_mod_name) - else: - mod = spec['ec'].full_mod_name - - if short: - item = os.path.join('$%s' % var_name, spec['spec'][len(common_prefix) + 1:]) - else: - item = spec['spec'] - lines.append(dry_run_fmt % (ans, item, mod)) - - if short: - # insert after 'Dry run:' message - lines.insert(1, "%s=%s" % (var_name, common_prefix)) - silent = build_option('silent') - print_msg('\n'.join(lines), log=_log, silent=silent, prefix=False) - - def _dep_graph(fn, specs, silent=False): """ Create a dependency graph for the given easyconfigs. @@ -345,7 +184,7 @@ def dep_graph(*args, **kwargs): _log.error("%s\nerr: %s" % (msg, err)) -def get_paths_for(subdir="easyconfigs", robot_path=None): +def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None): """ Return a list of absolute paths where the specified subdir can be found, determined by the PYTHONPATH """ @@ -374,7 +213,7 @@ def get_paths_for(subdir="easyconfigs", robot_path=None): # look for desired subdirs for path in path_list: path = os.path.join(path, "easybuild", subdir) - _log.debug("Looking for easybuild/%s in path %s" % (subdir, path)) + _log.debug("Checking for easybuild/%s at %s" % (subdir, path)) try: if os.path.exists(path): paths.append(os.path.abspath(path)) @@ -385,6 +224,113 @@ def get_paths_for(subdir="easyconfigs", robot_path=None): return paths +def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_pr=False): + """Obtain alternative paths for easyconfig files.""" + # path where tweaked easyconfigs will be placed + tweaked_ecs_path = None + if tweaked_ecs: + tweaked_ecs_path = os.path.join(tmpdir, 'tweaked_easyconfigs') + + # path where files touched in PR will be downloaded to + pr_path = None + if from_pr: + pr_path = os.path.join(tmpdir, "files_pr%s" % from_pr) + + return tweaked_ecs_path, pr_path + + +def det_easyconfig_paths(orig_paths, from_pr=None, easyconfigs_pkg_paths=None): + """ + Determine paths to easyconfig files. + @param orig_paths: list of original easyconfig paths + @param from_pr: pull request number to fetch easyconfigs from + @param easyconfigs_pkg_paths: paths to installed easyconfigs package + """ + if easyconfigs_pkg_paths is None: + easyconfigs_pkg_paths = [] + + # list of specified easyconfig files + ec_files = orig_paths[:] + + if from_pr is not None: + pr_files = fetch_easyconfigs_from_pr(from_pr) + + if ec_files: + # replace paths for specified easyconfigs that are touched in PR + for i, ec_file in enumerate(ec_files): + for pr_file in pr_files: + if ec_file == os.path.basename(pr_file): + ec_files[i] = pr_file + else: + # if no easyconfigs are specified, use all the ones touched in the PR + ec_files = [path for path in pr_files if path.endswith('.eb')] + + if ec_files and easyconfigs_pkg_paths: + # look for easyconfigs with relative paths in easybuild-easyconfigs package, + # unless they were found at the given relative paths + + # determine which easyconfigs files need to be found, if any + ecs_to_find = [] + for idx, ec_file in enumerate(ec_files): + if ec_file == os.path.basename(ec_file) and not os.path.exists(ec_file): + ecs_to_find.append((idx, ec_file)) + _log.debug("List of easyconfig files to find: %s" % ecs_to_find) + + # find missing easyconfigs by walking paths with installed easyconfig files + for path in easyconfigs_pkg_paths: + _log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path)) + for (subpath, dirnames, filenames) in os.walk(path, topdown=True): + for idx, orig_path in ecs_to_find[:]: + if orig_path in filenames: + full_path = os.path.join(subpath, orig_path) + _log.info("Found %s in %s: %s" % (orig_path, path, full_path)) + ec_files[idx] = full_path + # if file was found, stop looking for it (first hit wins) + ecs_to_find.remove((idx, orig_path)) + + # stop os.walk insanity as soon as we have all we need (os.walk loop) + if not ecs_to_find: + break + + # ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk + dirnames[:] = [d for d in dirnames if d not in build_option('ignore_dirs')] + + # stop os.walk insanity as soon as we have all we need (outer loop) + if not ecs_to_find: + break + + # indicate that specified paths do not contain generated easyconfig files + return [(ec_file, False) for ec_file in ec_files] + + +def parse_easyconfigs(paths): + """ + Parse easyconfig files + @params paths: paths to easyconfigs + """ + easyconfigs = [] + generated_ecs = False + for (path, generated) in paths: + path = os.path.abspath(path) + # keep track of whether any files were generated + generated_ecs |= generated + if not os.path.exists(path): + _log.error("Can't find path %s" % path) + try: + ec_files = find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs')) + for ec_file in ec_files: + # only pass build specs when not generating easyconfig files + kwargs = {} + if not build_option('try_to_generate'): + kwargs['build_specs'] = build_option('build_specs') + ecs = process_easyconfig(ec_file, **kwargs) + easyconfigs.extend(ecs) + except IOError, err: + _log.error("Processing easyconfigs in path %s failed: %s" % (path, err)) + + return easyconfigs, generated_ecs + + def stats_to_str(stats): """ Pretty print build statistics to string. diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index c2bfbe8374..0047ae8a6d 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -31,7 +31,7 @@ @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) @author: Toon Willems (Ghent University) -@author: Fotis Georgatos (University of Luxembourg) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ import copy import glob @@ -42,11 +42,10 @@ from vsc.utils import fancylogger from vsc.utils.missing import nub -from easybuild.tools.build_log import print_error, print_msg, print_warning from easybuild.framework.easyconfig.easyconfig import EasyConfig, create_paths, process_easyconfig -from easybuild.framework.easyconfig.tools import resolve_dependencies from easybuild.tools.filetools import read_file, write_file from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version +from easybuild.tools.robot import resolve_dependencies from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME from easybuild.tools.utilities import quote_str @@ -167,7 +166,7 @@ def __repr__(self): for (key, val) in tweaks.items(): if isinstance(val, list): - regexp = re.compile(r"^\s*%s\s*=\s*(.*)$" % key, re.M) + regexp = re.compile(r"^(?P\s*%s)\s*=\s*(?P.*)$" % key, re.M) res = regexp.search(ectxt) if res: fval = [x for x in val if x != ''] # filter out empty strings @@ -176,53 +175,53 @@ def __repr__(self): # - input starting with comma (empty head list element) => append # - no empty head/tail list element => overwrite if val[0] == '': - newval = "%s + %s" % (res.group(1), fval) + newval = "%s + %s" % (res.group('val'), fval) _log.debug("Appending %s to %s" % (fval, key)) elif val[-1] == '': - newval = "%s + %s" % (fval, res.group(1)) + newval = "%s + %s" % (fval, res.group('val')) _log.debug("Prepending %s to %s" % (fval, key)) else: newval = "%s" % fval _log.debug("Overwriting %s with %s" % (key, fval)) - ectxt = regexp.sub("%s = %s # tweaked by EasyBuild (was: %s)" % (key, newval, res.group(1)), ectxt) + ectxt = regexp.sub("%s = %s" % (res.group('key'), newval), ectxt) _log.info("Tweaked %s list to '%s'" % (key, newval)) else: - additions.append("%s = %s # added by EasyBuild" % (key, val)) + additions.append("%s = %s" % (key, val)) tweaks.pop(key) # add parameters or replace existing ones for (key, val) in tweaks.items(): - regexp = re.compile(r"^\s*%s\s*=\s*(.*)$" % key, re.M) + regexp = re.compile(r"^(?P\s*%s)\s*=\s*(?P.*)$" % key, re.M) _log.debug("Regexp pattern for replacing '%s': %s" % (key, regexp.pattern)) res = regexp.search(ectxt) if res: # only tweak if the value is different diff = True try: - _log.debug("eval(%s): %s" % (res.group(1), eval(res.group(1)))) - diff = not eval(res.group(1)) == val + _log.debug("eval(%s): %s" % (res.group('val'), eval(res.group('val')))) + diff = eval(res.group('val')) != val except (NameError, SyntaxError): # if eval fails, just fall back to string comparison - _log.debug("eval failed for \"%s\", falling back to string comparison against \"%s\"..." % (res.group(1), val)) - diff = not res.group(1) == val + tup = (res.group('val'), val) + _log.debug("eval failed for \"%s\", falling back to string comparison against \"%s\"..." % tup) + diff = res.group('val') != val if diff: - ectxt = regexp.sub("%s = %s # tweaked by EasyBuild (was: %s)" % (key, quote_str(val), res.group(1)), ectxt) + ectxt = regexp.sub("%s = %s" % (res.group('key'), quote_str(val)), ectxt) _log.info("Tweaked '%s' to '%s'" % (key, quote_str(val))) else: additions.append("%s = %s" % (key, quote_str(val))) if additions: - _log.info("Adding additional parameters to tweaked easyconfig file: %s") - ectxt += "\n\n# added by EasyBuild as dictated by command line options\n" - ectxt += '\n'.join(additions) + '\n' + _log.info("Adding additional parameters to tweaked easyconfig file: %s" % additions) + ectxt = '\n'.join([ectxt] + additions) _log.debug("Contents of tweaked easyconfig file:\n%s" % ectxt) # come up with suiting file name for tweaked easyconfig file if none was specified - if not target_fn: + if target_fn is None: fn = None try: # obtain temporary filename @@ -255,11 +254,14 @@ def __repr__(self): def pick_version(req_ver, avail_vers): """Pick version based on an optionally desired version and available versions. - If a desired version is specifed, the most recent version that is less recent - than the desired version will be picked; else, the most recent version will be picked. + If a desired version is specifed, the most recent version that is less recent than or equal to + the desired version will be picked; else, the most recent version will be picked. - This function returns both the version to be used, which is equal to the desired version + This function returns both the version to be used, which is equal to the required version if it was specified, and the version picked that matches that closest. + + @param req_ver: required version + @param avail_vers: list of available versions """ if not avail_vers: @@ -268,20 +270,18 @@ def pick_version(req_ver, avail_vers): selected_ver = None if req_ver: # if a desired version is specified, - # retain the most recent version that's less recent than the desired version - + # retain the most recent version that's less recent or equal than the desired version ver = req_ver if len(avail_vers) == 1: selected_ver = avail_vers[0] else: - retained_vers = [v for v in avail_vers if v < LooseVersion(ver)] + retained_vers = [v for v in avail_vers if v <= LooseVersion(ver)] if retained_vers: selected_ver = retained_vers[-1] else: # if no versions are available that are less recent, take the least recent version selected_ver = sorted([LooseVersion(v) for v in avail_vers])[0] - else: # if no desired version is specified, just use last version ver = avail_vers[-1] @@ -290,6 +290,26 @@ def pick_version(req_ver, avail_vers): return (ver, selected_ver) +def find_matching_easyconfigs(name, installver, paths): + """ + Find easyconfigs that match specified name/installversion in specified list of paths. + + @param name: software name + @param installver: software install version (which includes version, toolchain, versionprefix/suffix, ...) + @param paths: list of paths to search easyconfigs in + """ + ec_files = [] + for path in paths: + patterns = create_paths(path, name, installver) + for pattern in patterns: + more_ec_files = filter(os.path.isfile, glob.glob(pattern)) + _log.debug("Including files that match glob pattern '%s': %s" % (pattern, more_ec_files)) + ec_files.extend(more_ec_files) + + # only retain unique easyconfig paths + return nub(ec_files) + + def select_or_generate_ec(fp, paths, specs): """ Select or generate an easyconfig file with the given requirements, from existing easyconfig files. @@ -319,7 +339,6 @@ def select_or_generate_ec(fp, paths, specs): handled_params = ['name'] # find ALL available easyconfig files for specified software - ec_files = [] cfg = { 'version': '*', 'toolchain': {'name': DUMMY_TOOLCHAIN_NAME, 'version': '*'}, @@ -327,10 +346,8 @@ def select_or_generate_ec(fp, paths, specs): 'versionsuffix': '*', } installver = det_full_ec_version(cfg) - for path in paths: - patterns = create_paths(path, name, installver) - for pattern in patterns: - ec_files.extend(glob.glob(pattern)) + ec_files = find_matching_easyconfigs(name, installver, paths) + _log.debug("Unique ec_files: %s" % ec_files) # we need at least one config file to start from if len(ec_files) == 0: @@ -347,10 +364,6 @@ def select_or_generate_ec(fp, paths, specs): if len(ec_files) == 0: _log.error("No easyconfig files found for software %s, and no templates available. I'm all out of ideas." % name) - # only retain unique easyconfig files - ec_files = nub(ec_files) - _log.debug("Unique ec_files: %s" % ec_files) - ecs_and_files = [(EasyConfig(f, validate=False), f) for f in ec_files] # TOOLCHAIN NAME @@ -511,7 +524,7 @@ def unique(l): for (key, val) in specs.items(): if key in selected_ec._config: # values must be equal to have a full match - if not selected_ec[key] == val: + if selected_ec[key] != val: match = False else: # if we encounter a key that is not set in the selected easyconfig, we don't have a full match @@ -525,7 +538,7 @@ def unique(l): # GENERATE # if no file path was specified, generate a file name - if not fp: + if fp is None: cfg = { 'version': ver, 'toolchain': {'name': tcname, 'version': tcver}, @@ -543,7 +556,7 @@ def unique(l): return (True, fp) -def obtain_ec_for(specs, paths, fp): +def obtain_ec_for(specs, paths, fp=None): """ Obtain an easyconfig file to the given specifications. @@ -563,31 +576,6 @@ def obtain_ec_for(specs, paths, fp): if not paths: _log.error("No paths to look for easyconfig files, specify a path with --robot.") - # create glob patterns based on supplied info - - # figure out the install version - cfg = { - 'version': specs.get('version', '*'), - 'toolchain': { - 'name': specs.get('toolchain_name', '*'), - 'version': specs.get('toolchain_version', '*'), - }, - 'versionprefix': specs.get('versionprefix', '*'), - 'versionsuffix': specs.get('versionsuffix', '*'), - } - installver = det_full_ec_version(cfg) - - # find easyconfigs that match a pattern - easyconfig_files = [] - for path in paths: - patterns = create_paths(path, specs['name'], installver) - for pattern in patterns: - easyconfig_files.extend(glob.glob(pattern)) - - cnt = len(easyconfig_files) - - _log.debug("List of obtained easyconfig files (%d): %s" % (cnt, easyconfig_files)) - # select best easyconfig, or try to generate one that fits the requirements res = select_or_generate_ec(fp, paths, specs) @@ -595,26 +583,3 @@ def obtain_ec_for(specs, paths, fp): return res else: _log.error("No easyconfig found for requested software, and also failed to generate one.") - - -def obtain_path(specs, paths, try_to_generate=False, exit_on_error=True, silent=False): - """Obtain a path for an easyconfig that matches the given specifications.""" - - # if no easyconfig files/paths were provided, but we did get a software name, - # we can try and find a suitable easyconfig ourselves, or generate one if we can - (generated, fn) = obtain_ec_for(specs, paths, None) - if not generated: - return (fn, generated) - else: - # if an easyconfig was generated, make sure we're allowed to use it - if try_to_generate: - print_msg("Generated an easyconfig file %s, going to use it now..." % fn, silent=silent) - return (fn, generated) - else: - try: - os.remove(fn) - except OSError, err: - print_warning("Failed to remove generated easyconfig file %s: %s" % (fn, err)) - print_error(("Unable to find an easyconfig for the given specifications: %s; " - "to make EasyBuild try to generate a matching easyconfig, " - "use the --try-X options ") % specs, log=_log, exit_on_error=exit_on_error) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index af5b17672b..a50781e9b3 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -37,7 +37,7 @@ import os from easybuild.tools.config import build_path -from easybuild.tools.filetools import run_cmd +from easybuild.tools.run import run_cmd class Extension(object): diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index 1308561fb9..a7a5ba6488 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -58,8 +58,7 @@ def extra_options(extra_vars=None): extra_vars = {} if not isinstance(extra_vars, dict): - _log.deprecated("Obtained value of type '%s' for extra_vars, should be 'dict'" % type(extra_vars), '2.0') - extra_vars = dict(extra_vars) + _log.nosupport("Obtained value of type '%s' for extra_vars, should be 'dict'" % type(extra_vars), '2.0') extra_vars.update({ 'options': [{}, "Dictionary with extension options.", CUSTOM], diff --git a/easybuild/main.py b/easybuild/main.py index b250876641..f1ab88af41 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -33,43 +33,68 @@ @author: Jens Timmerman (Ghent University) @author: Toon Willems (Ghent University) @author: Ward Poelmans (Ghent University) -@author: Fotis Georgatos (University of Luxembourg) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ import copy import os -import subprocess import sys -import tempfile import traceback -from vsc.utils import fancylogger -from vsc.utils.missing import any # IMPORTANT this has to be the first easybuild import as it customises the logging # expect missing log output when this not the case! -from easybuild.tools.build_log import EasyBuildError, print_msg, print_error +from easybuild.tools.build_log import EasyBuildError, init_logging, print_msg, print_error, stop_logging import easybuild.tools.config as config import easybuild.tools.options as eboptions from easybuild.framework.easyblock import EasyBlock, build_and_install_one -from easybuild.framework.easyconfig.easyconfig import process_easyconfig -from easybuild.framework.easyconfig.tools import dep_graph, get_paths_for, print_dry_run -from easybuild.framework.easyconfig.tools import resolve_dependencies, skip_available -from easybuild.framework.easyconfig.tweak import obtain_path, tweak -from easybuild.tools.config import get_repository, module_classes, get_repositorypath, set_tmpdir -from easybuild.tools.filetools import cleanup, find_easyconfigs, search_file, write_file -from easybuild.tools.github import fetch_easyconfigs_from_pr +from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR +from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, dep_graph, det_easyconfig_paths +from easybuild.framework.easyconfig.tools import get_paths_for, parse_easyconfigs, skip_available +from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak +from easybuild.tools.config import get_repository, get_repositorypath, set_tmpdir +from easybuild.tools.filetools import cleanup, write_file from easybuild.tools.options import process_software_build_specs -from easybuild.tools.parallelbuild import build_easyconfigs_in_parallel +from easybuild.tools.robot import det_robot_path, dry_run, resolve_dependencies, search_easyconfigs +from easybuild.tools.parallelbuild import submit_jobs from easybuild.tools.repository.repository import init_repository -from easybuild.tools.testing import create_test_report, post_easyconfigs_pr_test_report, upload_test_report_as_gist -from easybuild.tools.testing import regtest, session_module_list, session_state -from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME -from easybuild.tools.version import this_is_easybuild # from a single location +from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_module_list, session_state +from easybuild.tools.version import this_is_easybuild _log = None +def log_start(eb_command_line, eb_tmpdir): + """Log startup info.""" + _log.info(this_is_easybuild()) + + # log used command line + _log.info("Command line: %s" % (' '.join(eb_command_line))) + + _log.info("Using %s as temporary directory" % eb_tmpdir) + + +def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=False): + """Find easyconfigs by build specifications.""" + generated, ec_file = obtain_ec_for(build_specs, robot_path, None) + if generated: + if try_to_generate: + print_msg("Generated an easyconfig file %s, going to use it now..." % ec_file, silent=testing) + else: + # (try to) cleanup + try: + os.remove(ec_file) + except OSError, err: + _log.warning("Failed to remove generated easyconfig file %s: %s" % (ec_file, err)) + + # don't use a generated easyconfig unless generation was requested (using a --try-X option) + _log.error(("Unable to find an easyconfig for the given specifications: %s; " + "to make EasyBuild try to generate a matching easyconfig, " + "use the --try-X options ") % build_specs) + + return [(ec_file, generated)] + + def build_and_install_software(ecs, init_session_state, exit_on_failure=True): """Build and install software for all provided parsed easyconfig files.""" # obtain a copy of the starting environment so each build can start afresh @@ -118,23 +143,12 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): def main(testing_data=(None, None, None)): """ - Main function: - @arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options - This function will: - - read easyconfig - - build software + Main function: parse command line options, and act accordingly. + @param testing_data: tuple with command line arguments, log file and boolean indicating whether or not to build """ - # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() - # disallow running EasyBuild as root - if os.getuid() == 0: - sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n" - "That's not wise, so let's end this here.\n" - "Exiting.\n") - sys.exit(1) - # steer behavior when testing main testing = testing_data[0] is not None args, logfile, do_build = testing_data @@ -143,8 +157,6 @@ def main(testing_data=(None, None, None)): eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args - eb_config = eb_go.generate_cmd_line(add_default=True) - init_session_state.update({'easybuild_configuration': eb_config}) # set umask (as early as possible) if options.umask is not None: @@ -155,226 +167,87 @@ def main(testing_data=(None, None, None)): eb_tmpdir = set_tmpdir(options.tmpdir) # initialise logging for main - if options.logtostdout: - fancylogger.logToScreen(enable=True, stdout=True) - else: - if logfile is None: - # mkstemp returns (fd,filename), fd is from os.open, not regular open! - fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-') - os.close(fd) + global _log + _log, logfile = init_logging(logfile, logtostdout=options.logtostdout, testing=testing) - fancylogger.logToFile(logfile) - print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing) + # disallow running EasyBuild as root + if os.getuid() == 0: + _log.error("You seem to be running EasyBuild with root privileges which is not wise, so let's end this here.") - global _log - _log = fancylogger.getLogger(fname=False) + # log startup info + eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args + log_start(eb_cmd_line, eb_tmpdir) if options.umask is not None: _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask))) - # hello world! - _log.info(this_is_easybuild()) - - # how was EB called? - eb_command_line = eb_go.generate_cmd_line() + eb_go.args - _log.info("Command line: %s" % (" ".join(eb_command_line))) - - _log.info("Using %s as temporary directory" % eb_tmpdir) - - if not options.robot is None: - if options.robot: - _log.info("Using robot path(s): %s" % options.robot) - else: - _log.error("No robot paths specified, and unable to determine easybuild-easyconfigs install path.") - - # do not pass options.robot, it's not a list instance (and it shouldn't be modified) - robot_path = [] - if options.robot: - robot_path = list(options.robot) - - # determine easybuild-easyconfigs package install path - easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path) - # keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs - easyconfigs_pkg_full_paths = easyconfigs_paths[:] - if not easyconfigs_paths: - _log.warning("Failed to determine install path for easybuild-easyconfigs package.") - # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) - # specified robot paths are preferred over installed easyconfig files - # --try-X and --dep-graph both require --robot, so enable it with path of installed easyconfigs - if robot_path or try_to_generate or options.dep_graph: - robot_path.extend(easyconfigs_paths) - easyconfigs_paths = robot_path[:] - _log.info("Extended list of robot paths with paths for installed easyconfigs: %s" % robot_path) - - # prepend robot path with location where tweaked easyconfigs will be placed - tweaked_ecs_path = None - if try_to_generate and build_specs: - tweaked_ecs_path = os.path.join(eb_tmpdir, 'tweaked_easyconfigs') - robot_path.insert(0, tweaked_ecs_path) - - # initialise the easybuild configuration - config.init(options, eb_go.get_options_by_section('config')) - - # building a dependency graph implies force, so that all dependencies are retained - # and also skips validation of easyconfigs (e.g. checking os dependencies) - retain_all_deps = False - if options.dep_graph: - _log.info("Enabling force to generate dependency graph.") - options.force = True - retain_all_deps = True - - if options.dep_graph or options.dry_run or options.dry_run_short: - options.ignore_osdeps = True - - pr_path = None - if options.from_pr: - # extend robot search path with location where files touch in PR will be downloaded to - pr_path = os.path.join(eb_tmpdir, "files_pr%s" % options.from_pr) - robot_path.insert(0, pr_path) - _log.info("Prepended list of robot search paths with %s: %s" % (pr_path, robot_path)) - - config.init_build_options({ - 'aggregate_regtest': options.aggregate_regtest, - 'allow_modules_tool_mismatch': options.allow_modules_tool_mismatch, - 'check_osdeps': not options.ignore_osdeps, - 'filter_deps': options.filter_deps, - 'cleanup_builddir': options.cleanup_builddir, - 'command_line': eb_command_line, - 'debug': options.debug, - 'dry_run': options.dry_run or options.dry_run_short, - 'easyblock': options.easyblock, - 'experimental': options.experimental, - 'force': options.force, - 'github_user': options.github_user, - 'group': options.group, - 'hidden': options.hidden, - 'ignore_dirs': options.ignore_dirs, - 'modules_footer': options.modules_footer, - 'only_blocks': options.only_blocks, - 'optarch': options.optarch, - 'recursive_mod_unload': options.recursive_module_unload, - 'regtest_output_dir': options.regtest_output_dir, - 'retain_all_deps': retain_all_deps, + # determine robot path + # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs + tweaked_ecs = try_to_generate and build_specs + tweaked_ecs_path, pr_path = alt_easyconfig_paths(eb_tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr) + auto_robot = try_to_generate or options.dep_graph or options.search or options.search_short + robot_path = det_robot_path(options.robot_paths, tweaked_ecs_path, pr_path, auto_robot=auto_robot) + _log.debug("Full robot path: %s" % robot_path) + + # configure & initialize build options + config_options_dict = eb_go.get_options_by_section('config') + build_options = { + 'build_specs': build_specs, + 'command_line': eb_cmd_line, + 'pr_path': pr_path, 'robot_path': robot_path, - 'sequential': options.sequential, 'silent': testing, - 'set_gid_bit': options.set_gid_bit, - 'skip': options.skip, - 'skip_test_cases': options.skip_test_cases, - 'sticky_bit': options.sticky_bit, - 'stop': options.stop, - 'suffix_modules_path': options.suffix_modules_path, - 'test_report_env_filter': options.test_report_env_filter, - 'umask': options.umask, - 'valid_module_classes': module_classes(), + 'try_to_generate': try_to_generate, 'valid_stops': [x[0] for x in EasyBlock.get_steps()], - 'validate': not options.force, - }) + } + # initialise the EasyBuild configuration & build options + config.init(options, config_options_dict) + config.init_build_options(build_options=build_options, cmdline_options=options) - # obtain list of loaded modules, build options must be initialized first - modlist = session_module_list() + # update session state + eb_config = eb_go.generate_cmd_line(add_default=True) + modlist = session_module_list(testing=testing) # build options must be initialized first before 'module list' works + init_session_state.update({'easybuild_configuration': eb_config}) init_session_state.update({'module_list': modlist}) _log.debug("Initial session state: %s" % init_session_state) - # search for easyconfigs - if options.search or options.search_short: - search_path = [os.getcwd()] - if easyconfigs_paths: - search_path = easyconfigs_paths - query = options.search or options.search_short - ignore_dirs = config.build_option('ignore_dirs') - silent = config.build_option('silent') - search_file(search_path, query, short=not options.search, ignore_dirs=ignore_dirs, silent=silent) - - paths = [] - if len(orig_paths) == 0: - if options.from_pr: - pr_files = fetch_easyconfigs_from_pr(options.from_pr, path=pr_path, github_user=options.github_user) - paths = [(path, False) for path in pr_files if path.endswith('.eb')] - elif 'name' in build_specs: - paths = [obtain_path(build_specs, easyconfigs_paths, try_to_generate=try_to_generate, - exit_on_error=not testing)] + # search for easyconfigs, if a query is specified + query = options.search or options.search_short + if query: + search_easyconfigs(query, short=not options.search) + + # determine easybuild-easyconfigs package install path + easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR) + if not easyconfigs_pkg_paths: + _log.warning("Failed to determine install path for easybuild-easyconfigs package.") + + # determine paths to easyconfigs + paths = det_easyconfig_paths(orig_paths, options.from_pr, easyconfigs_pkg_paths) + if not paths: + if 'name' in build_specs: + # try to obtain or generate an easyconfig file via build specifications if a software name is provided + paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing) elif not any([options.aggregate_regtest, options.search, options.search_short, options.regtest]): print_error(("Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), - log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) - else: - # look for easyconfigs with relative paths in easybuild-easyconfigs package, - # unless they were found at the given relative paths - if easyconfigs_pkg_full_paths: - # determine which easyconfigs files need to be found, if any - ecs_to_find = [] - for idx, orig_path in enumerate(orig_paths): - if orig_path == os.path.basename(orig_path) and not os.path.exists(orig_path): - ecs_to_find.append((idx, orig_path)) - _log.debug("List of easyconfig files to find: %s" % ecs_to_find) - - # find missing easyconfigs by walking paths with installed easyconfig files - for path in easyconfigs_pkg_full_paths: - _log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path)) - for (subpath, dirnames, filenames) in os.walk(path, topdown=True): - for idx, orig_path in ecs_to_find[:]: - if orig_path in filenames: - full_path = os.path.join(subpath, orig_path) - _log.info("Found %s in %s: %s" % (orig_path, path, full_path)) - orig_paths[idx] = full_path - # if file was found, stop looking for it (first hit wins) - ecs_to_find.remove((idx, orig_path)) - - # stop os.walk insanity as soon as we have all we need (os.walk loop) - if len(ecs_to_find) == 0: - break - - # ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk - dirnames[:] = [d for d in dirnames if not d in options.ignore_dirs] - - # stop os.walk insanity as soon as we have all we need (paths loop) - if len(ecs_to_find) == 0: - break - - # indicate that specified paths do not contain generated easyconfig files - paths = [(path, False) for path in orig_paths] - + log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") - if paths: - ec_paths = [path[0] for path in paths] - else: # fallback: easybuild-easyconfigs install path - ec_paths = easyconfigs_pkg_full_paths - regtest_ok = regtest(ec_paths) - + # fallback: easybuild-easyconfigs install path + regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 # read easyconfig files - easyconfigs = [] - generated_ecs = False - for (path, generated) in paths: - path = os.path.abspath(path) - # keep track of whether any files were generated - generated_ecs |= generated - if not os.path.exists(path): - print_error("Can't find path %s" % path) - - try: - ec_files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs) - for ec_file in ec_files: - # only pass build specs when not generating easyconfig files - if try_to_generate: - ecs = process_easyconfig(ec_file) - else: - ecs = process_easyconfig(ec_file, build_specs=build_specs) - easyconfigs.extend(ecs) - except IOError, err: - _log.error("Processing easyconfigs in path %s failed: %s" % (path, err)) + easyconfigs, generated_ecs = parse_easyconfigs(paths) # tweak obtained easyconfig files, if requested # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail @@ -382,25 +255,31 @@ def main(testing_data=(None, None, None)): if try_to_generate and build_specs and not generated_ecs: easyconfigs = tweak(easyconfigs, build_specs, targetdir=tweaked_ecs_path) - # before building starts, take snapshot of environment (watch out -t option!) - os.chdir(os.environ['PWD']) - # dry_run: print all easyconfigs and dependencies, and whether they are already built if options.dry_run or options.dry_run_short: - print_dry_run(easyconfigs, short=not options.dry_run, build_specs=build_specs) + txt = dry_run(easyconfigs, short=not options.dry_run, build_specs=build_specs) + print_msg(txt, log=_log, silent=testing, prefix=False) + # cleanup and exit after dry run, searching easyconfigs or submitting regression test if any([options.dry_run, options.dry_run_short, options.regtest, options.search, options.search_short]): cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # skip modules that are already installed unless forced if not options.force: - easyconfigs = skip_available(easyconfigs, testing=testing) + retained_ecs = skip_available(easyconfigs) + if not testing: + for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]: + print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name']) + easyconfigs = retained_ecs # determine an order that will allow all specs in the set to build if len(easyconfigs) > 0: - print_msg("resolving dependencies ...", log=_log, silent=testing) - ordered_ecs = resolve_dependencies(easyconfigs, build_specs=build_specs) + if options.robot: + print_msg("resolving dependencies ...", log=_log, silent=testing) + ordered_ecs = resolve_dependencies(easyconfigs, build_specs=build_specs) + else: + ordered_ecs = easyconfigs else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = [] @@ -411,27 +290,11 @@ def main(testing_data=(None, None, None)): dep_graph(options.dep_graph, ordered_ecs) sys.exit(0) - # submit build as job(s) and exit + # submit build as job(s), clean up and exit if options.job: - curdir = os.getcwd() - - # the options to ignore (help options can't reach here) - ignore_opts = ['robot', 'job'] - - # generate_cmd_line returns the options in form --longopt=value - opts = [x for x in eb_go.generate_cmd_line() if not x.split('=')[0] in ['--%s' % y for y in ignore_opts]] - - quoted_opts = subprocess.list2cmdline(opts) - - command = "unset TMPDIR && cd %s && eb %%(spec)s %s" % (curdir, quoted_opts) - _log.info("Command template for jobs: %s" % command) + job_info_txt = submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing) if not testing: - jobs = build_easyconfigs_in_parallel(command, ordered_ecs) - txt = ["List of submitted jobs:"] - txt.extend(["%s (%s): %s" % (job.name, job.module, job.jobid) for job in jobs]) - txt.append("(%d jobs submitted)" % len(jobs)) - - print_msg("Submitted parallel build jobs, exiting now: %s" % '\n'.join(txt), log=_log) + print_msg("Submitted parallel build jobs, exiting now: %s" % job_info_txt) cleanup(logfile, eb_tmpdir, testing) sys.exit(0) @@ -449,24 +312,10 @@ def main(testing_data=(None, None, None)): repo = init_repository(get_repository(), get_repositorypath()) repo.cleanup() - # report back in PR in case of testing - if options.upload_test_report: - msg = success_msg + " (%d easyconfigs in this PR)" % len(paths) - test_report = create_test_report(msg, ecs_with_res, init_session_state, pr_nr=options.from_pr, gist_log=True) - if options.from_pr: - # upload test report to gist and issue a comment in the PR to notify - msg = post_easyconfigs_pr_test_report(options.from_pr, test_report, success_msg, init_session_state, overall_success) - print_msg(msg) - else: - # only upload test report as a gist - gist_url = upload_test_report_as_gist(test_report) - print_msg("Test report uploaded to %s" % gist_url) - else: - test_report = create_test_report(success_msg, ecs_with_res, init_session_state) - _log.debug("Test report: %s" % test_report) - if options.dump_test_report is not None: - write_file(options.dump_test_report, test_report) - _log.info("Test report dumped to %s" % options.dump_test_report) + # dump/upload overall test report + test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state) + if test_report_msg is not None: + print_msg(test_report_msg) print_msg(success_msg, log=_log, silent=testing) @@ -475,17 +324,14 @@ def main(testing_data=(None, None, None)): if 'original_spec' in ec and os.path.isfile(ec['spec']): os.remove(ec['spec']) - # cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir path) - if options.logtostdout: - fancylogger.logToScreen(enable=False, stdout=True) - else: - fancylogger.logToFile(logfile, enable=False) + # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir path) + stop_logging(logfile, logtostdout=options.logtostdout) if overall_success: cleanup(logfile, eb_tmpdir, testing) + if __name__ == "__main__": try: main() except EasyBuildError, e: - sys.stderr.write('ERROR: %s\n' % e.msg) - sys.exit(1) + print_error(e.msg) diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index 91deb59323..3c8cc97141 100755 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -431,7 +431,7 @@ def main(): info('') info("By default, EasyBuild will install software to $HOME/.local/easybuild.") info("To install software with EasyBuild to %s, make sure $EASYBUILD_INSTALLPATH is set accordingly." % install_path) - info("See https://github.com/hpcugent/easybuild/wiki/Configuration for details on configuring EasyBuild.") + info("See http://easybuild.readthedocs.org/en/latest/Configuration.html for details on configuring EasyBuild.") # template easyconfig file for EasyBuild EB_EC_FILE = """ diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py new file mode 100755 index 0000000000..14e26a6a14 --- /dev/null +++ b/easybuild/scripts/clean_gists.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +## +# Copyright 2014 Ward Poelmans +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +This script cleans up old gists created by easybuild. It checks if the gists was +created from a pull-request and if that PR is closed/merged, it will delete the gist. +You need a github token for this. The script uses the same username and token +as easybuild. Optionally, you can specify a different github username. + +@author: Ward Poelmans +""" + + +import re + +from vsc.utils import fancylogger +from vsc.utils.generaloption import simple_option +from vsc.utils.rest import RestClient +from easybuild.tools.github import GITHUB_API_URL, HTTP_STATUS_OK, GITHUB_EASYCONFIGS_REPO +from easybuild.tools.github import GITHUB_EB_MAIN, fetch_github_token +from easybuild.tools.options import EasyBuildOptions + +HTTP_DELETE_OK = 204 + + +def main(): + """the main function""" + fancylogger.logToScreen(enable=True, stdout=True) + fancylogger.setLogLevelInfo() + + options = { + 'github-user': ('Your github username to use', None, 'store', None, 'g'), + 'closed-pr': ('Delete all gists from closed pull-requests', None, 'store_true', True, 'p'), + 'all': ('Delete all gists from Easybuild ', None, 'store_true', False, 'a'), + 'orphans': ('Delete all gists without a pull-request', None, 'store_true', False, 'o'), + } + + go = simple_option(options) + log = go.log + + if not (go.options.all or go.options.closed_pr or go.options.orphans): + log.error("Please tell me what to do?") + + if go.options.github_user is None: + eb_go = EasyBuildOptions(envvar_prefix='EASYBUILD', go_args=[]) + username = eb_go.options.github_user + log.debug("Fetch github username from easybuild, found: %s", username) + else: + username = go.options.github_user + + if username is None: + log.error("Could not find a github username") + else: + log.info("Using username = %s", username) + + token = fetch_github_token(username) + + gh = RestClient(GITHUB_API_URL, username=username, token=token) + # ToDo: add support for pagination + status, gists = gh.gists.get(per_page=100) + + if status != HTTP_STATUS_OK: + log.error("Failed to get a lists of gists for user %s: error code %s, message = %s", + username, status, gists) + else: + log.info("Found %s gists", len(gists)) + + regex = re.compile(r"(EasyBuild test report|EasyBuild log for failed build).*?(?:PR #(?P[0-9]+))?\)?$") + + pr_cache = {} + num_deleted = 0 + + for gist in gists: + if not gist["description"]: + continue + re_pr_num = regex.search(gist["description"]) + delete_gist = False + + if re_pr_num: + log.debug("Found a Easybuild gist (id=%s)", gist["id"]) + pr_num = re_pr_num.group("PR") + if go.options.all: + delete_gist = True + elif pr_num and go.options.closed_pr: + log.debug("Found Easybuild test report for PR #%s", pr_num) + + if pr_num not in pr_cache: + status, pr = gh.repos[GITHUB_EB_MAIN][GITHUB_EASYCONFIGS_REPO].pulls[pr_num].get() + if status != HTTP_STATUS_OK: + log.error("Failed to get pull-request #%s: error code %s, message = %s", + pr_num, status, pr) + pr_cache[pr_num] = pr["state"] + + if pr_cache[pr_num] == "closed": + log.debug("Found report from closed PR #%s (id=%s)", pr_num, gist["id"]) + delete_gist = True + + elif not pr_num and go.options.orphans: + log.debug("Found Easybuild test report without PR (id=%s)", gist["id"]) + delete_gist = True + + if delete_gist: + status, del_gist = gh.gists[gist["id"]].delete() + + if status != HTTP_DELETE_OK: + log.error("Unable to remove gist (id=%s): error code %s, message = %s", + gist["id"], status, del_gist) + else: + log.info("Delete gist with id=%s", gist["id"]) + num_deleted += 1 + + log.info("Deleted %s gists", num_deleted) + + +if __name__ == '__main__': + main() diff --git a/easybuild/scripts/generate_software_list.py b/easybuild/scripts/generate_software_list.py index 0183ff2c5e..94018d05d6 100644 --- a/easybuild/scripts/generate_software_list.py +++ b/easybuild/scripts/generate_software_list.py @@ -126,12 +126,13 @@ log.info("found valid easyconfig %s" % ec) if not ec.name in names: log.info("found new software package %s" % ec) + ec.easyblock = None # check if an easyblock exists - module = get_easyblock_class(None, name=ec.name).__module__.split('.')[-1] - if module != "configuremake": - ec.easyblock = module - else: - ec.easyblock = None + ebclass = get_easyblock_class(None, name=ec.name, default_fallback=False) + if ebclass is not None: + module = ebclass.__module__.split('.')[-1] + if module != "configuremake": + ec.easyblock = module configs.append(ec) names.append(ec.name) except Exception, err: diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py index 8c25f14135..0d57f13f35 100755 --- a/easybuild/scripts/mk_tmpl_easyblock_for.py +++ b/easybuild/scripts/mk_tmpl_easyblock_for.py @@ -121,7 +121,7 @@ import easybuild.tools.toolchain as toolchain %(parent_import)s from easybuild.framework.easyconfig import CUSTOM, MANDATORY -from easybuild.tools.filetools import run_cmd +from easybuild.tools.run import run_cmd class %(class_name)s(%(parent)s): @@ -136,11 +136,10 @@ def __init__(self, *args, **kwargs): @staticmethod def extra_options(): \"\"\"Custom easyconfig parameters for %(name)s.\"\"\" - - extra_vars = [ - ('mandatory_extra_param', ['default value', "short description", MANDATORY]), - ('optional_extra_param', ['default value', "short description", CUSTOM]), - ] + extra_vars = { + 'mandatory_extra_param': ['default value', "short description", MANDATORY], + 'optional_extra_param': ['default value', "short description", CUSTOM], + } return %(parent)s.extra_options(extra_vars) def configure_step(self): @@ -208,8 +207,8 @@ def make_module_extra(self): txt = super(%(class_name)s, self).make_module_extra() - txt += self.moduleGenerator.set_environment("VARIABLE", 'value') - txt += self.moduleGenerator.prepend_paths("PATH_VAR", ['path1', 'path2']) + txt += self.module_generator.set_environment("VARIABLE", 'value') + txt += self.module_generator.prepend_paths("PATH_VAR", ['path1', 'path2']) return txt """ diff --git a/easybuild/toolchains/cgmpich.py b/easybuild/toolchains/cgmpich.py index a85f500257..af3a4e9dbf 100644 --- a/easybuild/toolchains/cgmpich.py +++ b/easybuild/toolchains/cgmpich.py @@ -38,4 +38,3 @@ class Cgmpich(ClangGcc, Mpich): """Compiler toolchain with Clang, GFortran and MPICH.""" NAME = 'cgmpich' - COMPILER_MODULE_NAME = ['ClangGCC'] diff --git a/easybuild/toolchains/cgmvapich2.py b/easybuild/toolchains/cgmvapich2.py index d3a4b596c2..61a764c40e 100644 --- a/easybuild/toolchains/cgmvapich2.py +++ b/easybuild/toolchains/cgmvapich2.py @@ -38,4 +38,3 @@ class Cgmvapich2(ClangGcc, Mvapich2): """Compiler toolchain with Clang, GFortran and MVAPICH2.""" NAME = 'cgmvapich2' - COMPILER_MODULE_NAME = ['ClangGCC'] diff --git a/easybuild/toolchains/cgompi.py b/easybuild/toolchains/cgompi.py index 9fe8105313..b39c6a0c23 100644 --- a/easybuild/toolchains/cgompi.py +++ b/easybuild/toolchains/cgompi.py @@ -38,4 +38,3 @@ class Cgompi(ClangGcc, OpenMPI): """Compiler toolchain with Clang, GFortran and OpenMPI.""" NAME = 'cgompi' - COMPILER_MODULE_NAME = ['ClangGCC'] diff --git a/easybuild/toolchains/compiler/inteliccifort.py b/easybuild/toolchains/compiler/inteliccifort.py index c224278eeb..3804343c6d 100644 --- a/easybuild/toolchains/compiler/inteliccifort.py +++ b/easybuild/toolchains/compiler/inteliccifort.py @@ -56,7 +56,7 @@ class IntelIccIfort(Compiler): COMPILER_UNIQUE_OPTION_MAP = { 'i8': 'i8', 'r8': 'r8', - 'optarch': 'xHOST', + 'optarch': 'xHost', 'openmp': 'openmp', # both -openmp/-fopenmp are valid for enabling OpenMP 'strict': ['fp-speculation=strict', 'fp-model strict'], 'precise': ['fp-model precise'], @@ -69,8 +69,8 @@ class IntelIccIfort(Compiler): } COMPILER_OPTIMAL_ARCHITECTURE_OPTION = { - systemtools.INTEL : 'xHOST', - systemtools.AMD : 'msse3', + systemtools.INTEL : 'xHost', + systemtools.AMD : 'xHost', } COMPILER_CC = 'icc' diff --git a/easybuild/toolchains/fft/intelfftw.py b/easybuild/toolchains/fft/intelfftw.py index 12f35035b9..a6eb1e1eaa 100644 --- a/easybuild/toolchains/fft/intelfftw.py +++ b/easybuild/toolchains/fft/intelfftw.py @@ -33,7 +33,7 @@ from easybuild.toolchains.fft.fftw import Fftw from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.utilities import all, any + class IntelFFTW(Fftw): """FFTW wrapper functionality of Intel MKL""" diff --git a/easybuild/toolchains/gimpi.py b/easybuild/toolchains/gimpi.py new file mode 100644 index 0000000000..0ac2345266 --- /dev/null +++ b/easybuild/toolchains/gimpi.py @@ -0,0 +1,38 @@ +## +# Copyright 2012-2014 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gimpi compiler toolchain (includes GCC and Intel MPI). + +@author: Stijn De Weirdt (Ghent University) +@author: Kenneth Hoste (Ghent University) +""" + +from easybuild.toolchains.compiler.gcc import Gcc +from easybuild.toolchains.mpi.intelmpi import IntelMPI + + +class Gimpi(Gcc, IntelMPI): + """Compiler toolchain with GCC and Intel MPI.""" + NAME = 'gimpi' diff --git a/easybuild/toolchains/gmpich.py b/easybuild/toolchains/gmpich.py new file mode 100644 index 0000000000..4527db29f2 --- /dev/null +++ b/easybuild/toolchains/gmpich.py @@ -0,0 +1,37 @@ +## +# Copyright 2012-2014 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gmpich compiler toolchain (includes GCC and MPICH). + +@author: Kenneth Hoste (Ghent University) +""" + +from easybuild.toolchains.compiler.gcc import Gcc +from easybuild.toolchains.mpi.mpich import Mpich + + +class Gmpich(Gcc, Mpich): + """Compiler toolchain with GCC and MPICH.""" + NAME = 'gmpich' diff --git a/easybuild/toolchains/gmpolf.py b/easybuild/toolchains/gmpolf.py index 077290259e..35a0d0b899 100644 --- a/easybuild/toolchains/gmpolf.py +++ b/easybuild/toolchains/gmpolf.py @@ -36,9 +36,9 @@ from easybuild.toolchains.fft.fftw import Fftw from easybuild.toolchains.linalg.openblas import OpenBLAS from easybuild.toolchains.linalg.scalapack import ScaLAPACK -from easybuild.toolchains.mpi.mpich2 import Mpich2 +from easybuild.toolchains.mpi.mpich import Mpich -class Gmpolf(Gcc, Mpich2, OpenBLAS, ScaLAPACK, Fftw): - """Compiler toolchain with GCC, MPICH2, OpenBLAS, ScaLAPACK and FFTW.""" +class Gmpolf(Gcc, Mpich, OpenBLAS, ScaLAPACK, Fftw): + """Compiler toolchain with GCC, MPICH, OpenBLAS, ScaLAPACK and FFTW.""" NAME = 'gmpolf' diff --git a/easybuild/toolchains/gompic.py b/easybuild/toolchains/gompic.py index b6b0b26edb..0a331d5daf 100644 --- a/easybuild/toolchains/gompic.py +++ b/easybuild/toolchains/gompic.py @@ -26,7 +26,7 @@ EasyBuild support for gompic compiler toolchain (includes GCC and OpenMPI and CUDA). @author: Kenneth Hoste (Ghent University) -@author: Fotis Georgatos (University of Luxembourg) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ from easybuild.toolchains.gcccuda import GccCUDA diff --git a/easybuild/toolchains/gpsmpi.py b/easybuild/toolchains/gpsmpi.py new file mode 100644 index 0000000000..b1dc8e0ab5 --- /dev/null +++ b/easybuild/toolchains/gpsmpi.py @@ -0,0 +1,37 @@ +## +# Copyright 2012-2014 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gpsmpi compiler toolchain (includes GCC and Parastation MPICH). + +""" + +from easybuild.toolchains.gmpich import Gmpich + + +class Gpsmpi(Gmpich): + """Compiler toolchain with GCC and Parastation MPICH.""" + NAME = 'gpsmpi' + # Use Parastation naming + MPI_MODULE_NAME = ["psmpi"] diff --git a/easybuild/toolchains/gpsolf.py b/easybuild/toolchains/gpsolf.py new file mode 100644 index 0000000000..f454620df8 --- /dev/null +++ b/easybuild/toolchains/gpsolf.py @@ -0,0 +1,41 @@ +## +# Copyright 2013-2014 Ghent University +# +# This file is triple-licensed under GPLv2 (see below), MIT, and +# BSD three-clause licenses. +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gmpolf compiler toolchain (includes GCC, Parastation MPICH, OpenBLAS, LAPACK, ScaLAPACK and FFTW). + +""" + +from easybuild.toolchains.gpsmpi import Gpsmpi +from easybuild.toolchains.fft.fftw import Fftw +from easybuild.toolchains.linalg.openblas import OpenBLAS +from easybuild.toolchains.linalg.scalapack import ScaLAPACK + + +class Gpsolf(Gpsmpi, OpenBLAS, ScaLAPACK, Fftw): + """Compiler toolchain with GCC, Parastation MPICH, OpenBLAS, ScaLAPACK and FFTW.""" + NAME = 'gpsolf' diff --git a/easybuild/toolchains/iimpi.py b/easybuild/toolchains/iimpi.py old mode 100755 new mode 100644 diff --git a/easybuild/toolchains/impich.py b/easybuild/toolchains/impich.py new file mode 100644 index 0000000000..efe9c513a2 --- /dev/null +++ b/easybuild/toolchains/impich.py @@ -0,0 +1,38 @@ +## +# Copyright 2013-2014 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for impich compiler toolchain (includes Intel compilers (icc, ifort), MPICH. + +""" + +from easybuild.toolchains.compiler.inteliccifort import IntelIccIfort +from easybuild.toolchains.mpi.mpich import Mpich + + +class Impich(IntelIccIfort, Mpich): + """ + Compiler toolchain with Intel compilers (icc/ifort), MPICH. + """ + NAME = 'impich' diff --git a/easybuild/toolchains/impmkl.py b/easybuild/toolchains/impmkl.py index ba0f1521b6..383753d1db 100644 --- a/easybuild/toolchains/impmkl.py +++ b/easybuild/toolchains/impmkl.py @@ -23,20 +23,19 @@ # along with EasyBuild. If not, see . ## """ -EasyBuild support for impmkl compiler toolchain (includes Intel compilers (icc, ifort), MPICH2, +EasyBuild support for impmkl compiler toolchain (includes Intel compilers (icc, ifort), MPICH, Intel Math Kernel Library (MKL) , and Intel FFTW wrappers. @author: Kenneth Hoste (Ghent University) """ -from easybuild.toolchains.compiler.inteliccifort import IntelIccIfort +from easybuild.toolchains.impich import Impich from easybuild.toolchains.fft.intelfftw import IntelFFTW -from easybuild.toolchains.mpi.mpich2 import Mpich2 from easybuild.toolchains.linalg.intelmkl import IntelMKL -class Impmkl(IntelIccIfort, Mpich2, IntelMKL, IntelFFTW): +class Impmkl(Impich, IntelMKL, IntelFFTW): """ - Compiler toolchain with Intel compilers (icc/ifort), MPICH2, + Compiler toolchain with Intel compilers (icc/ifort), MPICH, Intel Math Kernel Library (MKL) and Intel FFTW wrappers. """ NAME = 'impmkl' diff --git a/easybuild/toolchains/intel-para.py b/easybuild/toolchains/intel-para.py new file mode 100644 index 0000000000..35e23da99a --- /dev/null +++ b/easybuild/toolchains/intel-para.py @@ -0,0 +1,42 @@ +## +# Copyright 2012-2013 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for intel compiler toolchain (includes Intel compilers (icc, ifort), Parastation MPICH, +Intel Math Kernel Library (MKL), and Intel FFTW wrappers). + +""" + +from easybuild.toolchains.ipsmpi import Ipsmpi +from easybuild.toolchains.fft.intelfftw import IntelFFTW +from easybuild.toolchains.linalg.intelmkl import IntelMKL + + +class IntelPara(Ipsmpi, IntelMKL, IntelFFTW): + """ + Compiler toolchain with Intel compilers (icc/ifort), Parastation MPICH, + Intel Math Kernel Library (MKL) and Intel FFTW wrappers. + """ + NAME = 'intel-para' + diff --git a/easybuild/toolchains/intel.py b/easybuild/toolchains/intel.py old mode 100755 new mode 100644 diff --git a/easybuild/toolchains/iompi.py b/easybuild/toolchains/iompi.py new file mode 100644 index 0000000000..fd757a3c92 --- /dev/null +++ b/easybuild/toolchains/iompi.py @@ -0,0 +1,40 @@ +## +# Copyright 2012-2014 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for iompi compiler toolchain (includes Intel compilers (icc, ifort) and OpenMPI. + +@author: Stijn De Weirdt (Ghent University) +@author: Kenneth Hoste (Ghent University) +""" + +from easybuild.toolchains.compiler.inteliccifort import IntelIccIfort +from easybuild.toolchains.mpi.openmpi import OpenMPI + + +class Iompi(IntelIccIfort, OpenMPI): + """ + Compiler toolchain with Intel compilers (icc/ifort) and OpenMPI. + """ + NAME = 'iompi' diff --git a/easybuild/toolchains/ipsmpi.py b/easybuild/toolchains/ipsmpi.py new file mode 100644 index 0000000000..8b99018284 --- /dev/null +++ b/easybuild/toolchains/ipsmpi.py @@ -0,0 +1,39 @@ +## +# Copyright 2012-2014 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for intel compiler toolchain (includes Intel compilers (icc, ifort), Parastation MPICH). + +""" + +from easybuild.toolchains.impich import Impich + + +class Ipsmpi(Impich): + """ + Compiler toolchain with Intel compilers (icc/ifort), Parastation MPICH. + """ + NAME = 'ipsmpi' + # Use Parastation naming + MPI_MODULE_NAME = ["psmpi"] diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py index 2f1186f6ee..af8cca4739 100644 --- a/easybuild/toolchains/linalg/intelmkl.py +++ b/easybuild/toolchains/linalg/intelmkl.py @@ -33,6 +33,11 @@ from easybuild.toolchains.compiler.inteliccifort import TC_CONSTANT_INTELCOMP from easybuild.toolchains.compiler.gcc import TC_CONSTANT_GCC +from easybuild.toolchains.mpi.intelmpi import TC_CONSTANT_INTELMPI +from easybuild.toolchains.mpi.mpich import TC_CONSTANT_MPICH +from easybuild.toolchains.mpi.mpich2 import TC_CONSTANT_MPICH2 +from easybuild.toolchains.mpi.mvapich2 import TC_CONSTANT_MVAPICH2 +from easybuild.toolchains.mpi.openmpi import TC_CONSTANT_OPENMPI from easybuild.tools.toolchain.linalg import LinAlg @@ -123,10 +128,14 @@ def _set_blas_variables(self): def _set_blacs_variables(self): mpimap = { - "OpenMPI": '_openmpi', - "IntelMPI": '_intelmpi', - "MVAPICH2": '_intelmpi', - "MPICH2":'', + TC_CONSTANT_OPENMPI: '_openmpi', + TC_CONSTANT_INTELMPI: '_intelmpi', + TC_CONSTANT_MVAPICH2: '_intelmpi', + # use intelmpi MKL blacs library for both MPICH v2 and v3 + # cfr. https://software.intel.com/en-us/articles/intel-mkl-link-line-advisor + # note: MKL link advisor uses 'MPICH' for MPICH v1 + TC_CONSTANT_MPICH2: '_intelmpi', + TC_CONSTANT_MPICH: '_intelmpi', } try: self.BLACS_LIB_MAP.update({'mpi': mpimap[self.MPI_FAMILY]}) diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index c08ffbf71c..1472e00e5e 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -33,6 +33,7 @@ """ import os import sys +import tempfile from copy import copy from vsc.utils import fancylogger @@ -48,6 +49,8 @@ # allow some experimental experimental code EXPERIMENTAL = False +DEPRECATED_DOC_URL = 'http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html' + class EasyBuildError(Exception): """ @@ -95,8 +98,13 @@ def experimental(self, msg, *args, **kwargs): def deprecated(self, msg, max_ver): """Print deprecation warning or raise an EasyBuildError, depending on max version allowed.""" + msg += "; see %s for more information" % DEPRECATED_DOC_URL fancylogger.FancyLogger.deprecated(self, msg, str(CURRENT_VERSION), max_ver, exception=EasyBuildError) + def nosupport(self, msg, ver): + """Print error message for no longer supported behaviour, and raise an EasyBuildError.""" + self.error("NO LONGER SUPPORTED since v%s: %s; see %s for more information" % (ver, msg, DEPRECATED_DOC_URL)) + def error(self, msg, *args, **kwargs): """Print error message and raise an EasyBuildError.""" newMsg = "EasyBuild crashed with an error %s: %s" % (self.caller_info(), msg) @@ -134,15 +142,36 @@ def exception(self, msg, *args): _init_easybuildlog = fancylogger.getLogger(fname=False) +def init_logging(logfile, logtostdout=False, testing=False): + """Initialize logging.""" + if logtostdout: + fancylogger.logToScreen(enable=True, stdout=True) + else: + if logfile is None: + # mkstemp returns (fd,filename), fd is from os.open, not regular open! + fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-') + os.close(fd) + + fancylogger.logToFile(logfile) + print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing) + + log = fancylogger.getLogger(fname=False) + + return log, logfile + + +def stop_logging(logfile, logtostdout=False): + """Stop logging.""" + if logtostdout: + fancylogger.logToScreen(enable=False, stdout=True) + fancylogger.logToFile(logfile, enable=False) + + def get_log(name=None): """ - Generate logger object + (NO LONGER SUPPORTED!) Generate logger object """ - # fname is always get_log, useless - log = fancylogger.getLogger(name, fname=False) - log.info("Logger started for %s." % name) - log.deprecated("get_log", "2.0") - return log + log.nosupport("Use of get_log function", '2.0') def print_msg(msg, log=None, silent=False, prefix=True): @@ -164,10 +193,9 @@ def print_error(message, log=None, exitCode=1, opt_parser=None, exit_on_error=Tr """ if exit_on_error: if not silent: - print_msg("ERROR: %s\n" % message) if opt_parser: opt_parser.print_shorthelp() - print_msg("ERROR: %s\n" % message) + sys.stderr.write("ERROR: %s\n" % message) sys.exit(exitCode) elif log is not None: log.error(message) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index d595b79803..60af87d556 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -51,13 +51,10 @@ _log = fancylogger.getLogger('config', fname=False) -# class constant to prepare migration to generaloption as only way of configuration (maybe for v2.X) -SUPPORT_OLDSTYLE = True - DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log") - - +DEFAULT_MNS = 'EasyBuildMNS' +DEFAULT_MODULES_TOOL = 'EnvironmentModulesC' DEFAULT_PATH_SUBDIRS = { 'buildpath': 'build', 'installpath': '', @@ -66,44 +63,80 @@ 'subdir_modules': 'modules', 'subdir_software': 'software', } - - -DEFAULT_BUILD_OPTIONS = { - 'aggregate_regtest': None, - 'allow_modules_tool_mismatch': False, - 'check_osdeps': True, - 'filter_deps': None, - 'cleanup_builddir': True, - 'command_line': None, - 'debug': False, - 'dry_run': False, - 'easyblock': None, - 'experimental': False, - 'force': False, - 'github_user': None, - 'group': None, - 'hidden': False, - 'ignore_dirs': None, - 'modules_footer': None, - 'only_blocks': None, - 'optarch': None, - 'recursive_mod_unload': False, - 'regtest_output_dir': None, - 'retain_all_deps': False, - 'robot_path': None, - 'sequential': False, - 'set_gid_bit': False, - 'silent': False, - 'skip': None, - 'skip_test_cases': False, - 'sticky_bit': False, - 'stop': None, - 'suffix_modules_path': None, - 'test_report_env_filter': None, - 'umask': None, - 'valid_module_classes': None, - 'valid_stops': None, - 'validate': True, +DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild") +DEFAULT_REPOSITORY = 'FileRepository' + + +# utility function for obtaining default paths +def mk_full_default_path(name, prefix=DEFAULT_PREFIX): + """Create full path, avoid '/' at the end.""" + args = [prefix] + path = DEFAULT_PATH_SUBDIRS[name] + if path: + args.append(path) + return os.path.join(*args) + +# build options that have a perfectly matching command line option, listed by default value +BUILD_OPTIONS_CMDLINE = { + None: [ + 'aggregate_regtest', + 'download_timeout', + 'dump_test_report', + 'easyblock', + 'filter_deps', + 'from_pr', + 'github_user', + 'group', + 'ignore_dirs', + 'modules_footer', + 'only_blocks', + 'optarch', + 'regtest_output_dir', + 'skip', + 'stop', + 'suffix_modules_path', + 'test_report_env_filter', + 'testoutput', + 'umask', + ], + False: [ + 'allow_modules_tool_mismatch', + 'debug', + 'experimental', + 'force', + 'hidden', + 'robot', + 'sequential', + 'set_gid_bit', + 'skip_test_cases', + 'sticky_bit', + 'upload_test_report', + ], + True: [ + 'cleanup_builddir', + ], +} +# build option that do not have a perfectly matching command line option +BUILD_OPTIONS_OTHER = { + None: [ + 'build_specs', + 'command_line', + 'pr_path', + 'robot_path', + 'valid_module_classes', + 'valid_stops', + ], + False: [ + 'dry_run', + 'recursive_mod_unload', + 'retain_all_deps', + 'silent', + 'try_to_generate', + ], + True: [ + 'check_osdeps', + 'validate', + ], } @@ -135,50 +168,13 @@ ] -OLDSTYLE_ENVIRONMENT_VARIABLES = { - 'build_path': 'EASYBUILDBUILDPATH', - 'config_file': 'EASYBUILDCONFIG', - 'install_path': 'EASYBUILDINSTALLPATH', - 'log_format': 'EASYBUILDLOGFORMAT', - 'log_dir': 'EASYBUILDLOGDIR', - 'source_path': 'EASYBUILDSOURCEPATH', - 'test_output_path': 'EASYBUILDTESTOUTPUT', -} - - -OLDSTYLE_NEWSTYLE_MAP = { - 'build_path': 'buildpath', - 'install_path': 'installpath', - 'log_dir': 'tmp_logdir', - 'config_file': 'config', - 'source_path': 'sourcepath', - 'log_format': 'logfile_format', - 'test_output_path': 'testoutput', - 'module_classes': 'moduleclasses', - 'repository_path': 'repositorypath', - 'modules_install_suffix': 'subdir_modules', - 'software_install_suffix': 'subdir_software', -} - - -def map_to_newstyle(adict): - """Map a dictionary with oldstyle keys to the new style.""" - res = {} - for key, val in adict.items(): - if key in OLDSTYLE_NEWSTYLE_MAP: - newkey = OLDSTYLE_NEWSTYLE_MAP.get(key) - _log.deprecated("oldstyle key %s usage found, replacing with newkey %s" % (key, newkey), "2.0") - key = newkey - res[key] = val - return res - - class ConfigurationVariables(FrozenDictKnownKeys): """This is a dict that supports legacy config names transparently.""" # singleton metaclass: only one instance is created __metaclass__ = Singleton + # list of known/required keys REQUIRED = [ 'config', 'prefix', @@ -195,21 +191,17 @@ class ConfigurationVariables(FrozenDictKnownKeys): 'modules_tool', 'module_naming_scheme', ] + KNOWN_KEYS = REQUIRED # KNOWN_KEYS must be defined for FrozenDictKnownKeys functionality - KNOWN_KEYS = nub(OLDSTYLE_NEWSTYLE_MAP.values() + REQUIRED) - - def get_items_check_required(self, no_missing=True): + def get_items_check_required(self): """ - For all REQUIRED, check if exists and return all key,value pairs. + For all known/required keys, check if exists and return all key/value pairs. no_missing: boolean, when True, will throw error message for missing values """ - missing = [x for x in self.REQUIRED if not x in self] + missing = [x for x in self.KNOWN_KEYS if not x in self] if len(missing) > 0: msg = 'Cannot determine value for configuration variables %s. Please specify it.' % missing - if no_missing: - self.log.error(msg) - else: - self.log.debug(msg) + self.log.error(msg) return self.items() @@ -220,94 +212,7 @@ class BuildOptions(FrozenDictKnownKeys): # singleton metaclass: only one instance is created __metaclass__ = Singleton - KNOWN_KEYS = DEFAULT_BUILD_OPTIONS.keys() - - -def get_user_easybuild_dir(): - """Return the per-user easybuild dir (e.g. to store config files)""" - oldpath = os.path.join(os.path.expanduser('~'), ".easybuild") - xdg_config_home = os.environ.get("XDG_CONFIG_HOME", os.path.join(os.path.expanduser('~'), ".config")) - newpath = os.path.join(xdg_config_home, "easybuild") - - if os.path.isdir(newpath): - return newpath - else: - _log.deprecated("The user easybuild dir has moved from %s to %s." % (oldpath, newpath), "2.0") - return oldpath - - -def get_default_oldstyle_configfile(): - """Get the default location of the oldstyle config file to be set as default in the options""" - # TODO these _log.debug here can't be controlled/set with the generaloption - # - check environment variable EASYBUILDCONFIG - # - next, check for an EasyBuild config in $HOME/.easybuild/config.py - # - last, use default config file easybuild_config.py in main.py directory - config_env_var = OLDSTYLE_ENVIRONMENT_VARIABLES['config_file'] - home_config_file = os.path.join(get_user_easybuild_dir(), "config.py") - if os.getenv(config_env_var): - _log.debug("Environment variable %s, so using that as config file." % config_env_var) - config_file = os.getenv(config_env_var) - elif os.path.exists(home_config_file): - config_file = home_config_file - _log.debug("Found EasyBuild configuration file at %s." % config_file) - else: - # this should be easybuild.tools.config, the default config file is - # part of framework in easybuild (ie in tool/..) - appPath = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - config_file = os.path.join(appPath, "easybuild_config.py") - _log.debug("Falling back to default config: %s" % config_file) - - _log.deprecated("get_default_oldstyle_configfile oldstyle configfile %s used" % config_file, "2.0") - - return config_file - - -def get_default_oldstyle_configfile_defaults(prefix=None): - """ - Return a dict with the defaults from the shipped legacy easybuild_config.py and/or environment variables - prefix: string, when provided, it used as prefix for the other defaults (where applicable) - """ - if prefix is None: - prefix = os.path.join(os.path.expanduser('~'), ".local", "easybuild") - - def mk_full_path(name): - """Create full path, avoid '/' at the end.""" - args = [prefix] - path = DEFAULT_PATH_SUBDIRS[name] - if path: - args.append(path) - return os.path.join(*args) - - # keys are the options dest - defaults = { - 'config': get_default_oldstyle_configfile(), - 'prefix': prefix, - 'buildpath': mk_full_path('buildpath'), - 'installpath': mk_full_path('installpath'), - 'sourcepath': mk_full_path('sourcepath'), - 'repository': 'FileRepository', - 'repositorypath': {'FileRepository': [mk_full_path('repositorypath')]}, - 'logfile_format': DEFAULT_LOGFILE_FORMAT[:], # make a copy - 'tmp_logdir': tempfile.gettempdir(), - 'moduleclasses': [x[0] for x in DEFAULT_MODULECLASSES], - 'subdir_modules': DEFAULT_PATH_SUBDIRS['subdir_modules'], - 'subdir_software': DEFAULT_PATH_SUBDIRS['subdir_software'], - 'modules_tool': 'EnvironmentModulesC', - 'module_naming_scheme': 'EasyBuildMNS', - } - - # sanity check - if not defaults['repository'] in defaults['repositorypath']: - _log.error('Failed to get repository path default for default %s' % (defaults['repository'])) - - _log.deprecated("get_default_oldstyle_configfile_defaults", "2.0") - - return defaults - - -def get_default_configfiles(): - """Return a list of default configfiles for tools.options/generaloption""" - return [os.path.join(get_user_easybuild_dir(), "config.cfg")] + KNOWN_KEYS = [k for kss in [BUILD_OPTIONS_CMDLINE, BUILD_OPTIONS_OTHER] for ks in kss.values() for k in ks] def get_pretend_installpath(): @@ -320,36 +225,7 @@ def init(options, config_options_dict): Gather all variables and check if they're valid Variables are read in this order of preference: generaloption > legacy environment > legacy config file """ - tmpdict = {} - - if SUPPORT_OLDSTYLE: - - _log.deprecated('oldstyle init with modifications to support oldstyle options', '2.0') - tmpdict.update(oldstyle_init(options.config)) - - # add the DEFAULT_MODULECLASSES as default (behavior is now that this extends the default list) - tmpdict['moduleclasses'] = nub(list(tmpdict.get('moduleclasses', [])) + - [x[0] for x in DEFAULT_MODULECLASSES]) - - # make sure we have new-style keys - tmpdict = map_to_newstyle(tmpdict) - - # all defaults are now set in generaloption - # distinguish between default generaloption values and values actually passed by generaloption - for dest in config_options_dict.keys(): - if not options._action_taken.get(dest, False): - if dest == 'installpath' and options.pretend: - # the installpath has been set by pretend option in postprocess - continue - # remove the default options if they are set in variables - # this way, all defaults are set - if dest in tmpdict: - _log.debug("Oldstyle support: no action for dest %s." % dest) - del config_options_dict[dest] - - # update the variables with the generaloption values - _log.debug("Updating config variables with generaloption dict %s" % config_options_dict) - tmpdict.update(config_options_dict) + tmpdict = copy.deepcopy(config_options_dict) # make sure source path is a list sourcepath = tmpdict['sourcepath'] @@ -365,12 +241,46 @@ def init(options, config_options_dict): _log.debug("Config variables: %s" % variables) -def init_build_options(build_options=None): +def init_build_options(build_options=None, cmdline_options=None): """Initialize build options.""" - # seed in defaults to make sure all build options are defined, and that build_option() doesn't fail on valid keys - bo = copy.deepcopy(DEFAULT_BUILD_OPTIONS) + + active_build_options = {} + + if cmdline_options is not None: + # building a dependency graph implies force, so that all dependencies are retained + # and also skips validation of easyconfigs (e.g. checking os dependencies) + retain_all_deps = False + if cmdline_options.dep_graph: + _log.info("Enabling force to generate dependency graph.") + cmdline_options.force = True + retain_all_deps = True + + if cmdline_options.dep_graph or cmdline_options.dry_run or cmdline_options.dry_run_short: + _log.info("Ignoring OS dependencies for --dep-graph/--dry-run") + cmdline_options.ignore_osdeps = True + + cmdline_build_option_names = [k for ks in BUILD_OPTIONS_CMDLINE.values() for k in ks] + active_build_options.update(dict([(key, getattr(cmdline_options, key)) for key in cmdline_build_option_names])) + # other options which can be derived but have no perfectly matching cmdline option + active_build_options.update({ + 'check_osdeps': not cmdline_options.ignore_osdeps, + 'dry_run': cmdline_options.dry_run or cmdline_options.dry_run_short, + 'recursive_mod_unload': cmdline_options.recursive_module_unload, + 'retain_all_deps': retain_all_deps, + 'validate': not cmdline_options.force, + 'valid_module_classes': module_classes(), + }) + if build_options is not None: - bo.update(build_options) + active_build_options.update(build_options) + + # seed in defaults to make sure all build options are defined, and that build_option() doesn't fail on valid keys + bo = {} + for build_options_by_default in [BUILD_OPTIONS_CMDLINE, BUILD_OPTIONS_OTHER]: + for default in build_options_by_default: + bo.update(dict([(opt, default) for opt in build_options_by_default[default]])) + bo.update(active_build_options) + # BuildOptions is a singleton, so any future calls to BuildOptions will yield the same instance return BuildOptions(bo) @@ -395,11 +305,8 @@ def source_paths(): def source_path(): - """ - Return the source path (deprecated) - """ - _log.deprecated("Use of source_path() is deprecated, use source_paths() instead.", '2.0') - return source_paths() + """NO LONGER SUPPORTED: use source_paths instead""" + _log.nosupport("source_path() is replaced by source_paths()", '2.0') def install_path(typ=None): @@ -408,25 +315,13 @@ def install_path(typ=None): - subdir 'software' for actual installation (default) - subdir 'modules' for environment modules (typ='mod') """ - variables = ConfigurationVariables() - if typ is None: typ = 'software' - if typ == 'mod': + elif typ == 'mod': typ = 'modules' - key = "subdir_%s" % typ - if key in variables: - suffix = variables[key] - else: - # TODO remove default setting. it should have been set through options - _log.deprecated('%s not set in config, returning default' % key, "2.0") - defaults = get_default_oldstyle_configfile_defaults() - try: - suffix = defaults[key] - except: - _log.error('install_path trying to get unknown suffix %s' % key) - + variables = ConfigurationVariables() + suffix = variables['subdir_%s' % typ] return os.path.join(variables['installpath'], suffix) @@ -462,16 +357,7 @@ def get_module_naming_scheme(): def log_file_format(return_directory=False): """Return the format for the logfile or the directory""" idx = int(not return_directory) - - variables = ConfigurationVariables() - if 'logfile_format' in variables: - res = variables['logfile_format'][idx] - else: - # TODO remove default setting. it should have been set through options - _log.deprecated('logfile_format not set in config, returning default', "2.0") - defaults = get_default_oldstyle_configfile_defaults() - res = defaults['logfile_format'][idx] - return res + return ConfigurationVariables()['logfile_format'][idx] def log_format(): @@ -491,16 +377,14 @@ def log_path(): def get_build_log_path(): """ - return temporary log directory + Return (temporary) directory for build log """ variables = ConfigurationVariables() - if 'tmp_logdir' in variables: - return variables['tmp_logdir'] + if variables['tmp_logdir'] is not None: + res = variables['tmp_logdir'] else: - # TODO remove default setting. it should have been set through options - _log.deprecated('tmp_logdir not set in config, returning default', "2.0") - defaults = get_default_oldstyle_configfile_defaults() - return defaults['tmp_logdir'] + res = tempfile.gettempdir() + return res def get_log_filename(name, version, add_salt=False): @@ -547,81 +431,12 @@ def module_classes(): """ Return list of module classes specified in config file. """ - variables = ConfigurationVariables() - if 'moduleclasses' in variables: - return variables['moduleclasses'] - else: - # TODO remove default setting. it should have been set through options - _log.deprecated('moduleclasses not set in config, returning default', "2.0") - defaults = get_default_oldstyle_configfile_defaults() - return defaults['moduleclasses'] + return ConfigurationVariables()['moduleclasses'] def read_environment(env_vars, strict=False): - """Depreacted location for read_environment, use easybuild.tools.environment""" - _log.deprecated("Deprecated location for read_environment, use easybuild.tools.environment", '2.0') - return _read_environment(env_vars, strict) - - -def oldstyle_init(filename, **kwargs): - """ - Gather all variables and check if they're valid - Variables are read in this order of preference: CLI option > environment > config file - """ - res = {} - _log.deprecated("oldstyle_init filename %s kwargs %s" % (filename, kwargs), "2.0") - - _log.debug('variables before oldstyle_init %s' % res) - res.update(oldstyle_read_configuration(filename)) # config file - _log.debug('variables after oldstyle_init read_configuration (%s) %s' % (filename, res)) - res.update(oldstyle_read_environment()) # environment - _log.debug('variables after oldstyle_init read_environment %s' % res) - if kwargs: - res.update(kwargs) # CLI options - _log.debug('variables after oldstyle_init kwargs (passed %s) %s' % (kwargs, res)) - - return res - - -def oldstyle_read_configuration(filename): - """ - Read variables from the config file - """ - _log.deprecated("oldstyle_read_configuration filename %s" % filename, "2.0") - - # import avail_repositories here to avoid cyclic dependencies - # this block of code is going to be removed in EB v2.0 - from easybuild.tools.repository.repository import avail_repositories - file_variables = avail_repositories(check_useable=False) - try: - execfile(filename, {}, file_variables) - except (IOError, SyntaxError), err: - _log.exception("Failed to read config file %s %s" % (filename, err)) - - return file_variables - - -def oldstyle_read_environment(env_vars=None, strict=False): - """ - Read variables from the environment - - strict=True enforces that all possible environment variables are found - """ - _log.deprecated(('Adapt code to use read_environment from easybuild.tools.utilities ' - 'and do not use oldstyle environment variables'), '2.0') - if env_vars is None: - env_vars = OLDSTYLE_ENVIRONMENT_VARIABLES - result = {} - for key in env_vars.keys(): - env_var = env_vars[key] - if env_var in os.environ: - result[key] = os.environ[env_var] - _log.deprecated("Found oldstyle environment variable %s for %s: %s" % (env_var, key, result[key]), "2.0") - elif strict: - _log.error("Can't determine value for %s. Environment variable %s is missing" % (key, env_var)) - else: - _log.debug("Old style env var %s not defined." % env_var) - - return result + """NO LONGER SUPPORTED: use read_environment from easybuild.tools.environment instead""" + _log.nosupport("read_environment has moved to easybuild.tools.environment", '2.0') def set_tmpdir(tmpdir=None): diff --git a/easybuild/tools/deprecated/__init__.py b/easybuild/tools/deprecated/__init__.py new file mode 100644 index 0000000000..8c26fd6795 --- /dev/null +++ b/easybuild/tools/deprecated/__init__.py @@ -0,0 +1,38 @@ +## +# Copyright 2009-2014 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +This declares the namespace for the tools.deprecated submodule of EasyBuild, +which provides deprecated functionality. + +@author: Stijn De Weirdt (Ghent University) +@author: Dries Verdegem (Ghent University) +@author: Kenneth Hoste (Ghent University) +@author: Pieter De Baets (Ghent University) +@author: Jens Timmerman (Ghent University) +""" +from pkgutil import extend_path + +# we're not the only ones in this namespace +__path__ = extend_path(__path__, __name__) #@ReservedAssignment diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py new file mode 100644 index 0000000000..c8033be791 --- /dev/null +++ b/easybuild/tools/docs.py @@ -0,0 +1,162 @@ +# # +# Copyright 2009-2014 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Documentation-related functionality + +@author: Stijn De Weirdt (Ghent University) +@author: Dries Verdegem (Ghent University) +@author: Kenneth Hoste (Ghent University) +@author: Pieter De Baets (Ghent University) +@author: Jens Timmerman (Ghent University) +@author: Toon Willems (Ghent University) +@author: Ward Poelmans (Ghent University) +""" +import copy + +from easybuild.framework.easyconfig.default import DEFAULT_CONFIG, HIDDEN, sorted_categories +from easybuild.framework.easyconfig.easyconfig import get_easyblock_class +from easybuild.tools.ordereddict import OrderedDict +from easybuild.tools.utilities import quote_str + + +FORMAT_RST = 'rst' +FORMAT_TXT = 'txt' + + +def avail_easyconfig_params_rst(title, grouped_params): + """ + Compose overview of available easyconfig parameters, in RST format. + """ + def det_col_width(entries, title): + """Determine column width based on column title and list of entries.""" + return max(map(len, entries + [title])) + + # main title + lines = [ + title, + '=' * len(title), + '', + ] + + for grpname in grouped_params: + # group section title + lines.append("%s parameters" % grpname) + lines.extend(['-' * len(lines[-1]), '']) + + name_title = "**Parameter name**" + descr_title = "**Description**" + dflt_title = "**Default value**" + + # figure out column widths + nw = det_col_width(grouped_params[grpname].keys(), name_title) + 4 # +4 for raw format ("``foo``") + dw = det_col_width([x[0] for x in grouped_params[grpname].values()], descr_title) + dfw = det_col_width([str(quote_str(x[1])) for x in grouped_params[grpname].values()], dflt_title) + + # 3 columns (name, description, default value), left-aligned, {c} is fill char + line_tmpl = "{0:{c}<%s} {1:{c}<%s} {2:{c}<%s}" % (nw, dw, dfw) + table_line = line_tmpl.format('', '', '', c='=', nw=nw, dw=dw, dfw=dfw) + + # table header + lines.append(table_line) + lines.append(line_tmpl.format(name_title, descr_title, dflt_title, c=' ')) + lines.append(line_tmpl.format('', '', '', c='-')) + + # table rows by parameter + for name, (descr, dflt) in sorted(grouped_params[grpname].items()): + rawname = '``%s``' % name + lines.append(line_tmpl.format(rawname, descr, str(quote_str(dflt)), c=' ')) + lines.append(table_line) + lines.append('') + + return '\n'.join(lines) + +def avail_easyconfig_params_txt(title, grouped_params): + """ + Compose overview of available easyconfig parameters, in plain text format. + """ + # main title + lines = [ + '%s:' % title, + '', + ] + + for grpname in grouped_params: + # group section title + lines.append(grpname.upper()) + lines.append('-' * len(lines[-1])) + + # determine width of 'name' column, to left-align descriptions + nw = max(map(len, grouped_params[grpname].keys())) + + # line by parameter + for name, (descr, dflt) in sorted(grouped_params[grpname].items()): + lines.append("{0:<{nw}} {1:} [default: {2:}]".format(name, descr, str(quote_str(dflt)), nw=nw)) + lines.append('') + + return '\n'.join(lines) + +def avail_easyconfig_params(easyblock, output_format): + """ + Compose overview of available easyconfig parameters, in specified format. + """ + params = copy.deepcopy(DEFAULT_CONFIG) + + # include list of extra parameters (if any) + extra_params = {} + app = get_easyblock_class(easyblock, default_fallback=False) + if app is not None: + extra_params = app.extra_options() + params.update(extra_params) + + # compose title + title = "Available easyconfig parameters" + if extra_params: + title += " (* indicates specific to the %s easyblock)" % app.__name__ + + # group parameters by category + grouped_params = OrderedDict() + for category in sorted_categories(): + # exclude hidden parameters + if category[1].upper() in [HIDDEN]: + continue + + grpname = category[1] + grouped_params[grpname] = {} + for name, (dflt, descr, cat) in params.items(): + if cat == category: + if name in extra_params: + # mark easyblock-specific parameters + name = '%s*' % name + grouped_params[grpname].update({name: (descr, dflt)}) + + if not grouped_params[grpname]: + del grouped_params[grpname] + + # compose output, according to specified format (txt, rst, ...) + avail_easyconfig_params_functions = { + FORMAT_RST: avail_easyconfig_params_rst, + FORMAT_TXT: avail_easyconfig_params_txt, + } + return avail_easyconfig_params_functions[output_format](title, grouped_params) diff --git a/easybuild/tools/environment.py b/easybuild/tools/environment.py index 073c43829d..384c65febd 100644 --- a/easybuild/tools/environment.py +++ b/easybuild/tools/environment.py @@ -102,7 +102,6 @@ def restore_env_vars(env_keys): """ Restore the environment by setting the keys in the env_keys dict again with their old value """ - for key in env_keys: if env_keys[key] is not None: _log.info("Restoring environment variable %s (value: %s)" % (key, env_keys[key])) @@ -150,3 +149,10 @@ def modify_env(old, new): _log.debug("Key in old environment found that is not in new one: %s (%s)" % (key, old[key])) os.unsetenv(key) del os.environ[key] + + +def restore_env(env): + """ + Restore active environment based on specified dictionary. + """ + modify_env(os.environ, env) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 91b417fddb..8b81d71d1c 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -32,17 +32,16 @@ @author: Jens Timmerman (Ghent University) @author: Toon Willems (Ghent University) @author: Ward Poelmans (Ghent University) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ -import errno import os import re import shutil import stat import time -import urllib +import urllib2 import zlib from vsc.utils import fancylogger -from vsc.utils.missing import all, any import easybuild.tools.environment as env from easybuild.tools.build_log import print_msg # import build_log must stay, to activate use of EasyBuildLog @@ -172,6 +171,15 @@ def write_file(path, txt, append=False): _log.error("Failed to write to %s: %s" % (path, err)) +def remove_file(path): + """Remove file at specified path.""" + try: + if os.path.exists(path): + os.remove(path) + except OSError, err: + _log.error("Failed to remove %s: %s", path, err) + + def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False): """ Given filename fn, try to extract in directory dest @@ -245,37 +253,54 @@ def det_common_path_prefix(paths): def download_file(filename, url, path): """Download a file from the given URL, to the specified path.""" - _log.debug("Downloading %s from %s to %s" % (filename, url, path)) + _log.debug("Trying to download %s from %s to %s", filename, url, path) + + timeout = build_option('download_timeout') + if timeout is None: + # default to 10sec timeout if none was specified + # default system timeout (used is nothing is specified) may be infinite (?) + timeout = 10 + _log.debug("Using timeout of %s seconds for initiating download" % timeout) # make sure directory exists basedir = os.path.dirname(path) mkdir(basedir, parents=True) + # try downloading, three times max. downloaded = False + max_attempts = 3 attempt_cnt = 0 - - # try downloading three times max. - while not downloaded and attempt_cnt < 3: - - (_, httpmsg) = urllib.urlretrieve(url, path) - - if httpmsg.type == "text/html" and not filename.endswith('.html'): - _log.warning("HTML file downloaded but not expecting it, so assuming invalid download.") - _log.debug("removing downloaded file %s from %s" % (filename, path)) - try: - os.remove(path) - except OSError, err: - _log.error("Failed to remove downloaded file:" % err) - else: - _log.info("Downloading file %s from url %s: done" % (filename, url)) + while not downloaded and attempt_cnt < max_attempts: + try: + # urllib2 does the right thing for http proxy setups, urllib does not! + url_fd = urllib2.urlopen(url, timeout=timeout) + _log.debug('response code for given url %s: %s' % (url, url_fd.getcode())) + write_file(path, url_fd.read()) + _log.info("Downloaded file %s from url %s to %s" % (filename, url, path)) downloaded = True - return path - - attempt_cnt += 1 - _log.warning("Downloading failed at attempt %s, retrying..." % attempt_cnt) - - # failed to download after multiple attempts - return None + url_fd.close() + except urllib2.HTTPError as err: + if 400 <= err.code <= 499: + _log.warning("URL %s was not found (HTTP response code %s), not trying again" % (url, err.code)) + break + else: + _log.warning("HTTPError occured while trying to download %s to %s: %s" % (url, path, err)) + attempt_cnt += 1 + except IOError as err: + _log.warning("IOError occurred while trying to download %s to %s: %s" % (url, path, err)) + attempt_cnt += 1 + except Exception, err: + _log.error("Unexpected error occurred when trying to download %s to %s: %s" % (url, path, err)) + + if not downloaded and attempt_cnt < max_attempts: + _log.info("Attempt %d of downloading %s to %s failed, trying again..." % (attempt_cnt, url, path)) + + if downloaded: + _log.info("Successful download of file %s from url %s to path %s" % (filename, url, path)) + return path + else: + _log.warning("Download of %s to %s failed, done trying" % (url, path)) + return None def find_easyconfigs(path, ignore_dirs=None): @@ -548,7 +573,11 @@ def det_patched_files(path=None, txt=None, omit_ab_prefix=False): patched_file = match.group('file') if not omit_ab_prefix and match.group('ab_prefix') is not None: patched_file = match.group('ab_prefix') + patched_file - patched_files.append(patched_file) + + if patched_file in ['/dev/null']: + _log.debug("Ignoring patched file %s" % patched_file) + else: + patched_files.append(patched_file) return patched_files @@ -647,11 +676,8 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None): def modify_env(old, new): - """ - Compares 2 os.environ dumps. Adapts final environment. - """ - _log.deprecated("moved modify_env to tools.environment", "2.0") - return env.modify_env(old, new) + """NO LONGER SUPPORTED: use modify_env from easybuild.tools.environment instead""" + _log.nosupport("moved modify_env to easybuild.tools.environment", "2.0") def convert_name(name, upper=False): @@ -978,19 +1004,17 @@ def decode_class_name(name): def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None): - """Legacy wrapper/placeholder for run.run_cmd""" - return run.run_cmd(cmd, log_ok=log_ok, log_all=log_all, simple=simple, - inp=inp, regexp=regexp, log_output=log_output, path=path) + """NO LONGER SUPPORTED: use run_cmd from easybuild.tools.run instead""" + _log.nosupport("run_cmd was moved from easybuild.tools.filetools to easybuild.tools.run", '2.0') def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None): - """Legacy wrapper/placeholder for run.run_cmd_qa""" - return run.run_cmd_qa(cmd, qa, no_qa=no_qa, log_ok=log_ok, log_all=log_all, - simple=simple, regexp=regexp, std_qa=std_qa, path=path) + """NO LONGER SUPPORTED: use run_cmd_qa from easybuild.tools.run instead""" + _log.nosupport("run_cmd_qa was moved from easybuild.tools.filetools to easybuild.tools.run", '2.0') def parse_log_for_error(txt, regExp=None, stdout=True, msg=None): - """Legacy wrapper/placeholder for run.parse_log_for_error""" - return run.parse_log_for_error(txt, regExp=regExp, stdout=stdout, msg=msg) + """NO LONGER SUPPORTED: use parse_log_for_error from easybuild.tools.run instead""" + _log.nosupport("parse_log_for_error was moved from easybuild.tools.filetools to easybuild.tools.run", '2.0') def det_size(path): diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index ae544e28b4..1ff47d1555 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -30,7 +30,6 @@ """ import base64 import os -import re import socket import tempfile import urllib @@ -56,6 +55,7 @@ _log.warning("Failed to import from 'vsc.utils.rest' Python module: %s" % err) HAVE_GITHUB_API = False +from easybuild.tools.config import build_option from easybuild.tools.filetools import det_patched_files, mkdir @@ -193,6 +193,11 @@ class GithubError(Exception): def fetch_easyconfigs_from_pr(pr, path=None, github_user=None): """Fetch patched easyconfig files for a particular PR.""" + if github_user is None: + github_user = build_option('github_user') + if path is None: + path = build_option('pr_path') + def download(url, path=None): """Download file from specified URL to specified path.""" if path is not None: @@ -246,7 +251,7 @@ def download(url, path=None): diff_txt = download(pr_data['diff_url']) patched_files = det_patched_files(txt=diff_txt, omit_ab_prefix=True) - _log.debug("List of patches files: %s" % patched_files) + _log.debug("List of patched files: %s" % patched_files) # obtain last commit # get all commits, increase to (max of) 100 per page diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 0c9f65253b..4a8c03e699 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -30,9 +30,10 @@ @author: Kenneth Hoste (Ghent University) @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) -@author: Fotis Georgatos (Uni.Lu) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ import os +import re import tempfile from vsc.utils import fancylogger @@ -47,7 +48,14 @@ _log = fancylogger.getLogger('module_generator', fname=False) -class ModuleGenerator: +class ModuleGenerator(object): + """ + Class for generating module files. + """ + + # chars we want to escape in the generated modulefiles + CHARS_TO_ESCAPE = ["$"] + def __init__(self, application, fake=False): self.fake = fake self.app = application @@ -231,6 +239,8 @@ def msg_on_load(self, msg): """ Add a message that should be printed when loading the module. """ + # escape any (non-escaped) characters with special meaning by prefixing them with a backslash + msg = re.sub(r'((? MPI//// namespace - tc_mpi_fullver = tc_mpi['version'] + tc_mpi['versionsuffix'] + tc_mpi_fullver = self.det_full_version(tc_mpi) subdir = os.path.join(MPI, tc_comp_name, tc_comp_ver, tc_mpi['name'], tc_mpi_fullver) return subdir @@ -123,17 +139,39 @@ def det_modpath_extensions(self, ec): Examples: Compiler/GCC/4.8.3 (for GCC/4.8.3 module), MPI/GCC/4.8.3/OpenMPI/1.6.5 (for OpenMPI/1.6.5 module) """ modclass = ec['moduleclass'] + tc_comps = det_toolchain_compilers(ec) + tc_comp_info = self.det_toolchain_compilers_name_version(tc_comps) paths = [] - if modclass == MODULECLASS_COMPILER: - if ec['name'] in ['icc', 'ifort']: - compdir = 'intel' + if modclass == MODULECLASS_COMPILER or ec['name'] in ['CUDA']: + # obtain list of compilers based on that extend $MODULEPATH in some way other than / + extend_comps = [] + # exclude GCC for which / is used as $MODULEPATH extension + excluded_comps = ['GCC'] + for comps in COMP_NAME_VERSION_TEMPLATES.keys(): + extend_comps.extend([comp for comp in comps.split(',') if comp not in excluded_comps]) + + comp_name_ver = None + if ec['name'] in extend_comps: + for key in COMP_NAME_VERSION_TEMPLATES: + if ec['name'] in key.split(','): + comp_name, comp_ver_tmpl = COMP_NAME_VERSION_TEMPLATES[key] + comp_versions = {ec['name']: self.det_full_version(ec)} + if ec['name'] == 'ifort': + # 'icc' key should be provided since it's the only one used in the template + comp_versions.update({'icc': self.det_full_version(ec)}) + if tc_comp_info is not None: + # also provide toolchain version for non-dummy toolchains + comp_versions.update({tc_comp_info[0]: tc_comp_info[1]}) + + comp_name_ver = [comp_name, comp_ver_tmpl % comp_versions] + break else: - compdir = ec['name'] - paths.append(os.path.join(COMPILER, compdir, ec['version'])) + comp_name_ver = [ec['name'], self.det_full_version(ec)] + + paths.append(os.path.join(COMPILER, *comp_name_ver)) + elif modclass == MODULECLASS_MPI: - tc_comps = det_toolchain_compilers(ec) - tc_comp_info = self.det_toolchain_compilers_name_version(tc_comps) if tc_comp_info is None: tup = (ec['toolchain'], ec['name'], ec['version']) error_msg = ("No compiler available in toolchain %s used to install MPI library %s v%s, " @@ -141,7 +179,7 @@ def det_modpath_extensions(self, ec): self.log.error(error_msg) else: tc_comp_name, tc_comp_ver = tc_comp_info - fullver = ec['version'] + ec['versionsuffix'] + fullver = self.det_full_version(ec) paths.append(os.path.join(MPI, tc_comp_name, tc_comp_ver, ec['name'], fullver)) return paths diff --git a/easybuild/tools/module_naming_scheme/utilities.py b/easybuild/tools/module_naming_scheme/utilities.py index 951d8015b3..8bd5053231 100644 --- a/easybuild/tools/module_naming_scheme/utilities.py +++ b/easybuild/tools/module_naming_scheme/utilities.py @@ -30,7 +30,7 @@ @author: Kenneth Hoste (Ghent University) @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) -@author: Fotis Georgatos (Uni.Lu) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ import os import string diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 6ab696a327..c8cd2c8914 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -42,12 +42,12 @@ from distutils.version import StrictVersion from subprocess import PIPE from vsc.utils import fancylogger -from vsc.utils.missing import get_subclasses, any +from vsc.utils.missing import get_subclasses from vsc.utils.patterns import Singleton from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, get_modules_tool, install_path -from easybuild.tools.environment import modify_env +from easybuild.tools.environment import restore_env from easybuild.tools.filetools import convert_name, mkdir, read_file, path_matches, which from easybuild.tools.module_naming_scheme import DEVEL_MODULE_SUFFIX from easybuild.tools.run import run_cmd @@ -137,12 +137,14 @@ class ModulesTool(object): __metaclass__ = Singleton - def __init__(self, mod_paths=None): + def __init__(self, mod_paths=None, testing=False): """ Create a ModulesTool object @param mod_paths: A list of paths where the modules can be located @type mod_paths: list """ + # this can/should be set to True during testing + self.testing = testing self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) self.mod_paths = None @@ -172,18 +174,14 @@ def __init__(self, mod_paths=None): self.check_module_function(allow_mismatch=build_option('allow_modules_tool_mismatch')) self.set_and_check_version() - # this can/should be set to True during testing - self.testing = False - def buildstats(self): """Return tuple with data to be included in buildstats""" return (self.__class__.__name__, self.cmd, self.version) @property def modules(self): - """Property providing access to deprecated 'modules' class variable.""" - self.log.deprecated("'modules' class variable is deprecated, just use load([])", '2.0') - return self._modules + """(NO LONGER SUPPORTED!) Property providing access to 'modules' class variable""" + self.log.nosupport("'modules' class variable is not supported anymore, use load([]) instead", '2.0') def set_and_check_version(self): """Get the module version, and check any requirements""" @@ -198,6 +196,14 @@ def set_and_check_version(self): if res: self.version = res.group('version') self.log.info("Found version %s" % self.version) + + # make sure version is a valid StrictVersion (e.g., 5.7.3.1 is invalid), + # and replace 'rc' by 'b', to make StrictVersion treat it as a beta-release + self.version = self.version.replace('rc', 'b') + if len(self.version.split('.')) > 3: + self.version = '.'.join(self.version.split('.')[:3]) + + self.log.info("Converted actual version to '%s'" % self.version) else: self.log.error("Failed to determine version from option '%s' output: %s" % (self.VERSION_OPTION, txt)) except (OSError), err: @@ -206,12 +212,7 @@ def set_and_check_version(self): if self.REQ_VERSION is None: self.log.debug('No version requirement defined.') else: - # make sure version is a valid StrictVersion (e.g., 5.7.3.1 is invalid), - # and replace 'rc' by 'b', to make StrictVersion treat it as a beta-release - check_ver = self.version.replace('rc', 'b') - if len(check_ver.split('.')) > 3: - check_ver = '.'.join(check_ver.split('.')[:3]) - if StrictVersion(check_ver) < StrictVersion(self.REQ_VERSION): + if StrictVersion(self.version) < StrictVersion(self.REQ_VERSION): msg = "EasyBuild requires v%s >= v%s (no rc), found v%s" self.log.error(msg % (self.__class__.__name__, self.REQ_VERSION, self.version)) else: @@ -229,11 +230,20 @@ def check_cmd_avail(self): def check_module_function(self, allow_mismatch=False, regex=None): """Check whether selected module tool matches 'module' function definition.""" - out, ec = run_cmd("type module", simple=False, log_ok=False, log_all=False) + if self.testing: + # grab 'module' function definition from environment if it's there; only during testing + if 'module' in os.environ: + out, ec = os.environ['module'], 0 + else: + out, ec = None, 1 + else: + out, ec = run_cmd("type module", simple=False, log_ok=False, log_all=False) + if regex is None: regex = r".*%s" % os.path.basename(self.cmd) mod_cmd_re = re.compile(regex, re.M) mod_details = "pattern '%s' (%s)" % (mod_cmd_re.pattern, self.__class__.__name__) + if ec == 0: if mod_cmd_re.search(out): self.log.debug("Found pattern '%s' in defined 'module' function." % mod_cmd_re.pattern) @@ -315,16 +325,19 @@ def check_module_path(self): self.use(mod_path) self.log.info("$MODULEPATH set based on list of module paths (via 'module use'): %s" % os.environ['MODULEPATH']) - def available(self, mod_name=None): + def available(self, mod_name=None, extra_args=None): """ Return a list of available modules for the given (partial) module name; use None to obtain a list of all available modules. @param mod_name: a (partial) module name for filtering (default: None) """ + if extra_args is None: + extra_args = [] if mod_name is None: mod_name = '' - mods = self.run_module('avail', mod_name) + args = ['avail'] + extra_args + [mod_name] + mods = self.run_module(*args) # sort list of modules in alphabetical order mods.sort(key=lambda m: m['mod_name']) @@ -356,9 +369,8 @@ def exist(self, mod_names): return mods_exist def exists(self, mod_name): - """Check if a module with the specified name exists.""" - self.log.deprecated("exists() is deprecated, use exist([]) instead", '2.0') - return self.exist([mod_name])[0] + """NO LONGER SUPPORTED: use exist method instead""" + self.log.nosupport("exists() is not supported anymore, use exist([]) instead", '2.0') def load(self, modules, mod_paths=None, purge=False, orig_env=None): """ @@ -377,7 +389,7 @@ def load(self, modules, mod_paths=None, purge=False, orig_env=None): self.purge() # restore original environment if provided if orig_env is not None: - modify_env(os.environ, orig_env) + restore_env(orig_env) # make sure $MODULEPATH is set correctly after purging self.check_module_path() @@ -394,8 +406,7 @@ def unload(self, modules=None): Unload all requested modules. """ if modules is None: - self.log.deprecated("Unloading modules listed in _modules class variable", '2.0') - modules = self._modules[:] + self.log.nosupport("Unloading modules listed in _modules class variable", '2.0') for mod in modules: self.run_module('unload', mod) @@ -456,15 +467,12 @@ def run_module(self, *args, **kwargs): args.insert(*self.TERSE_OPTION) module_path_key = None - original_module_path = None if 'mod_paths' in kwargs: module_path_key = 'mod_paths' elif 'modulePath' in kwargs: module_path_key = 'modulePath' if module_path_key is not None: - original_module_path = os.environ['MODULEPATH'] - os.environ['MODULEPATH'] = kwargs[module_path_key] - self.log.deprecated("Use of '%s' named argument in 'run_module'" % module_path_key, '2.0') + self.log.nosupport("Use of '%s' named argument in 'run_module'" % module_path_key, '2.0') self.log.debug('Current MODULEPATH: %s' % os.environ.get('MODULEPATH', '')) @@ -490,9 +498,6 @@ def run_module(self, *args, **kwargs): # stderr will contain text (just like the normal module command) (stdout, stderr) = proc.communicate() self.log.debug("Output of module command '%s': stdout: %s; stderr: %s" % (full_cmd, stdout, stderr)) - if original_module_path is not None: - os.environ['MODULEPATH'] = original_module_path - self.log.deprecated("Restoring $MODULEPATH back to what it was before running module command/.", '2.0') if kwargs.get('return_output', False): return stdout + stderr @@ -592,6 +597,8 @@ def modpath_extensions_for(self, mod_names): Determine dictionary with $MODULEPATH extensions for specified modules. Modules with an empty list of $MODULEPATH extensions are included. """ + self.log.debug("Determining $MODULEPATH extensions for modules %s" % mod_names) + # copy environment so we can restore it orig_env = os.environ.copy() @@ -601,6 +608,7 @@ def modpath_extensions_for(self, mod_names): useregex = re.compile(r"^\s*module\s+use\s+(\S+)", re.M) exts = useregex.findall(modtxt) + self.log.debug("Found $MODULEPATH extensions for %s: %s" % (mod_name, exts)) modpath_exts.update({mod_name: exts}) if exts: @@ -609,7 +617,7 @@ def modpath_extensions_for(self, mod_names): self.load([mod_name]) # restore original environment (modules may have been loaded above) - modify_env(os.environ, orig_env) + restore_env(orig_env) return modpath_exts @@ -658,40 +666,41 @@ def path_to_top_of_module_tree(self, top_paths, mod_name, full_mod_subdir, deps, modpath_exts = dict([(k, v) for k, v in self.modpath_extensions_for(deps).items() if v]) self.log.debug("Non-empty lists of module path extensions for dependencies: %s" % modpath_exts) - path = [] + mods_to_top = [] + full_mod_subdirs = [] for dep in modpath_exts: # if a $MODULEPATH extension is identical to where this module will be installed, we have a hit # use os.path.samefile when comparing paths to avoid issues with resolved symlinks full_modpath_exts = modpath_exts[dep] if path_matches(full_mod_subdir, full_modpath_exts): # full path to module subdir of dependency is simply path to module file without (short) module name - full_mod_subdir = self.modulefile_path(dep)[:-len(dep)-1] + dep_full_mod_subdir = self.modulefile_path(dep)[:-len(dep)-1] + full_mod_subdirs.append(dep_full_mod_subdir) - path.append(dep) - tup = (dep, full_mod_subdir, full_modpath_exts) + mods_to_top.append(dep) + tup = (dep, dep_full_mod_subdir, full_modpath_exts) self.log.debug("Found module to top of module tree: %s (subdir: %s, modpath extensions %s)" % tup) - # no need to continue further, we found the module that extends $MODULEPATH with module subdir - break - if full_modpath_exts: # load module for this dependency, since it may extend $MODULEPATH to make dependencies available # this is required to obtain the corresponding module file paths (via 'module show') self.load([dep]) - if path: - # remove retained dependency from the list, since we're climbing up the module tree - modpath_exts.pop(path[-1]) + # restore original environment (modules may have been loaded above) + restore_env(orig_env) + + path = mods_to_top[:] + if mods_to_top: + # remove retained dependencies from the list, since we're climbing up the module tree + remaining_modpath_exts = dict([m for m in modpath_exts.items() if not m[0] in mods_to_top]) - self.log.debug("Path to top from %s extended to %s, so recursing to find way to the top" % (mod_name, path)) - path.extend(self.path_to_top_of_module_tree(top_paths, path[-1], full_mod_subdir, None, - modpath_exts=modpath_exts)) + self.log.debug("Path to top from %s extended to %s, so recursing to find way to the top" % (mod_name, mods_to_top)) + for mod_name, full_mod_subdir in zip(mods_to_top, full_mod_subdirs): + path.extend(self.path_to_top_of_module_tree(top_paths, mod_name, full_mod_subdir, None, + modpath_exts=remaining_modpath_exts)) else: self.log.debug("Path not extended, we must have reached the top of the module tree") - # restore original environment (modules may have been loaded above) - modify_env(os.environ, orig_env) - self.log.debug("Path to top of module tree from %s: %s" % (mod_name, path)) return path @@ -808,7 +817,13 @@ def available(self, mod_name=None): @param name: a (partial) module name for filtering (default: None) """ - mods = super(Lmod, self).available(mod_name=mod_name) + extra_args = [] + if StrictVersion(self.version) >= StrictVersion('5.7.5'): + # make hidden modules visible for recent version of Lmod + extra_args = ['--show_hidden'] + + mods = super(Lmod, self).available(mod_name=mod_name, extra_args=extra_args) + # only retain actual modules, exclude module directories (which end with a '/') real_mods = [mod for mod in mods if not mod.endswith('/')] @@ -862,24 +877,22 @@ def get_software_root(name, with_env_var=False): """ Return the software root set for a particular software name. """ - environment_key = get_software_root_env_var_name(name) - newname = convert_name(name, upper=True) - legacy_key = "SOFTROOT%s" % newname + env_var = get_software_root_env_var_name(name) + legacy_key = "SOFTROOT%s" % convert_name(name, upper=True) - # keep on supporting legacy installations - if environment_key in os.environ: - env_var = environment_key - else: - env_var = legacy_key - if legacy_key in os.environ: - _log.deprecated("Legacy env var %s is being relied on!" % legacy_key, "2.0") + root = None + if env_var in os.environ: + root = os.getenv(env_var) - root = os.getenv(env_var) + elif legacy_key in os.environ: + _log.nosupport("Legacy env var %s is being relied on!" % legacy_key, "2.0") if with_env_var: - return (root, env_var) + res = (root, env_var) else: - return root + res = root + + return res def get_software_libdir(name, only_one=True, fs=None): @@ -925,18 +938,16 @@ def get_software_version(name): """ Return the software version set for a particular software name. """ - environment_key = get_software_version_env_var_name(name) - newname = convert_name(name, upper=True) - legacy_key = "SOFTVERSION%s" % newname + env_var = get_software_version_env_var_name(name) + legacy_key = "SOFTVERSION%s" % convert_name(name, upper=True) - # keep on supporting legacy installations - if environment_key in os.environ: - return os.getenv(environment_key) - else: - if legacy_key in os.environ: - _log.deprecated("Legacy env var %s is being relied on!" % legacy_key, "2.0") - return os.getenv(legacy_key) + version = None + if env_var in os.environ: + version = os.getenv(env_var) + elif legacy_key in os.environ: + _log.nosupport("Legacy env var %s is being relied on!" % legacy_key, "2.0") + return version def curr_module_paths(): """ @@ -963,7 +974,7 @@ def avail_modules_tools(): return class_dict -def modules_tool(mod_paths=None): +def modules_tool(mod_paths=None, testing=False): """ Return interface to modules tool (environment modules (C, Tcl), or Lmod) """ @@ -971,15 +982,12 @@ def modules_tool(mod_paths=None): modules_tool = get_modules_tool() if modules_tool is not None: modules_tool_class = avail_modules_tools().get(modules_tool) - return modules_tool_class(mod_paths=mod_paths) + return modules_tool_class(mod_paths=mod_paths, testing=testing) else: return None -# provide Modules class for backward compatibility (e.g., in easyblocks) class Modules(EnvironmentModulesC): - """Deprecated interface to modules tool.""" - + """NO LONGER SUPPORTED: interface to modules tool, use modules_tool from easybuild.tools.modules instead""" def __init__(self, *args, **kwargs): - _log.deprecated("modules.Modules class is now an abstract interface, use modules.modules_tool instead", "2.0") - super(Modules, self).__init__(*args, **kwargs) + _log.nosupport("modules.Modules class is now an abstract interface, use modules.modules_tool instead", '2.0') diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 4d15565caf..80fd4db664 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -40,18 +40,19 @@ from vsc.utils.missing import nub from easybuild.framework.easyblock import EasyBlock +from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR from easybuild.framework.easyconfig.constants import constant_documentation -from easybuild.framework.easyconfig.default import convert_to_help -from easybuild.framework.easyconfig.easyconfig import get_easyblock_class from easybuild.framework.easyconfig.format.pyheaderconfigobj import build_easyconfig_constants_dict from easybuild.framework.easyconfig.licenses import license_documentation from easybuild.framework.easyconfig.templates import template_documentation from easybuild.framework.easyconfig.tools import get_paths_for from easybuild.framework.extension import Extension from easybuild.tools import build_log, config, run # @UnusedImport make sure config is always initialized! -from easybuild.tools.config import get_default_configfiles, get_pretend_installpath -from easybuild.tools.config import get_default_oldstyle_configfile_defaults, DEFAULT_MODULECLASSES -from easybuild.tools.convert import ListOfStrings +from easybuild.tools.config import DEFAULT_LOGFILE_FORMAT, DEFAULT_MNS, DEFAULT_MODULES_TOOL, DEFAULT_MODULECLASSES +from easybuild.tools.config import DEFAULT_PATH_SUBDIRS, DEFAULT_PREFIX, DEFAULT_REPOSITORY +from easybuild.tools.config import get_pretend_installpath +from easybuild.tools.config import mk_full_default_path +from easybuild.tools.docs import FORMAT_RST, FORMAT_TXT, avail_easyconfig_params from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, fetch_github_token from easybuild.tools.modules import avail_modules_tools from easybuild.tools.module_naming_scheme import GENERAL_CLASS @@ -62,7 +63,10 @@ from easybuild.tools.version import this_is_easybuild from vsc.utils import fancylogger from vsc.utils.generaloption import GeneralOption -from vsc.utils.missing import any + + +XDG_CONFIG_HOME = os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), ".config")) +DEFAULT_CONFIGFILE = os.path.join(XDG_CONFIG_HOME, 'easybuild', 'config.cfg') class EasyBuildOptions(GeneralOption): @@ -70,21 +74,36 @@ class EasyBuildOptions(GeneralOption): VERSION = this_is_easybuild() DEFAULT_LOGLEVEL = 'INFO' - DEFAULT_CONFIGFILES = get_default_configfiles() + DEFAULT_CONFIGFILES = [DEFAULT_CONFIGFILE] ALLOPTSMANDATORY = False # allow more than one argument + def __init__(self, *args, **kwargs): + """Constructor.""" + + self.default_robot_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None) or [] + + # set up constants to seed into config files parser, by section + self.go_cfg_constants = { + self.DEFAULTSECT: { + 'DEFAULT_ROBOT_PATHS': (os.pathsep.join(self.default_robot_paths), + "List of default robot paths ('%s'-separated)" % os.pathsep), + } + } + + # update or define go_configfiles_initenv in named arguments to pass to parent constructor + go_cfg_initenv = kwargs.setdefault('go_configfiles_initenv', {}) + for section, constants in self.go_cfg_constants.items(): + constants = dict([(name, value) for (name, (value, _)) in constants.items()]) + go_cfg_initenv.setdefault(section, {}).update(constants) + + super(EasyBuildOptions, self).__init__(*args, **kwargs) + def basic_options(self): """basic runtime options""" all_stops = [x[0] for x in EasyBlock.get_steps()] strictness_options = [run.IGNORE, run.WARN, run.ERROR] - try: - default_robot_path = get_paths_for("easyconfigs", robot_path=None)[0] - except: - self.log.warning("basic_options: unable to determine default easyconfig path") - default_robot_path = False # False as opposed to None, since None is used for indicating that --robot was used - descr = ("Basic options", "Basic runtime options for EasyBuild.") opts = OrderedDict({ @@ -95,8 +114,10 @@ def basic_options(self): 'job': ("Submit the build as a job", None, 'store_true', False), 'logtostdout': ("Redirect main log to stdout", None, 'store_true', False, 'l'), 'only-blocks': ("Only build listed blocks", None, 'extend', None, 'b', {'metavar': 'BLOCKS'}), - 'robot': ("Path(s) to search for easyconfigs for missing dependencies (colon-separated)" , - None, 'store_or_None', default_robot_path, 'r', {'metavar': 'PATH'}), + 'robot': ("Enable dependency resolution, using easyconfigs in specified paths", + 'pathlist', 'store_or_None', [], 'r', {'metavar': 'PATH[%sPATH]' % os.pathsep}), + 'robot-paths': ("Additional paths to consider by robot for easyconfigs (--robot paths get priority)", + 'pathlist', 'store', self.default_robot_paths, {'metavar': 'PATH[%sPATH]' % os.pathsep}), 'skip': ("Skip existing software (useful for installing additional packages)", None, 'store_true', False, 'k'), 'stop': ("Stop the installation after certain step", 'choice', 'store_or_None', 'source', 's', all_stops), @@ -158,6 +179,7 @@ def override_options(self): 'cleanup-builddir': ("Cleanup build dir after successful installation.", None, 'store_true', True), 'deprecated': ("Run pretending to be (future) version, to test removal of deprecated code.", None, 'store', None), + 'download_timeout': ("Timeout for initiating downloads (in seconds)", None, 'store', None), 'easyblock': ("easyblock to use for processing the spec file or dumping the options", None, 'store', None, 'e', {'metavar': 'CLASS'}), 'experimental': ("Allow experimental code (with behaviour that can be changed or removed at any given time).", @@ -188,8 +210,6 @@ def config_options(self): # config options descr = ("Configuration options", "Configure EasyBuild behavior.") - oldstyle_defaults = get_default_oldstyle_configfile_defaults() - opts = OrderedDict({ 'avail-module-naming-schemes': ("Show all supported module naming schemes", None, 'store_true', False,), @@ -197,50 +217,45 @@ def config_options(self): None, "store_true", False,), 'avail-repositories': ("Show all repository types (incl. non-usable)", None, "store_true", False,), - 'buildpath': ("Temporary build path", None, 'store', oldstyle_defaults['buildpath']), + 'buildpath': ("Temporary build path", None, 'store', mk_full_default_path('buildpath')), 'ignore-dirs': ("Directory names to ignore when searching for files/dirs", 'strlist', 'store', ['.git', '.svn']), - 'installpath': ("Install path for software and modules", None, 'store', oldstyle_defaults['installpath']), - 'config': ("Path to EasyBuild config file", - None, 'store', oldstyle_defaults['config'], 'C'), + 'installpath': ("Install path for software and modules", None, 'store', mk_full_default_path('installpath')), + # purposely take a copy for the default logfile format 'logfile-format': ("Directory name and format of the log file", - 'strtuple', 'store', oldstyle_defaults['logfile_format'], {'metavar': 'DIR,FORMAT'}), + 'strtuple', 'store', DEFAULT_LOGFILE_FORMAT[:], {'metavar': 'DIR,FORMAT'}), 'module-naming-scheme': ("Module naming scheme", - 'choice', 'store', oldstyle_defaults['module_naming_scheme'], - sorted(avail_module_naming_schemes().keys())), + 'choice', 'store', DEFAULT_MNS, sorted(avail_module_naming_schemes().keys())), 'moduleclasses': (("Extend supported module classes " "(For more info on the default classes, use --show-default-moduleclasses)"), - None, 'extend', oldstyle_defaults['moduleclasses']), + None, 'extend', [x[0] for x in DEFAULT_MODULECLASSES]), 'modules-footer': ("Path to file containing footer to be added to all generated module files", None, 'store_or_None', None, {'metavar': "PATH"}), 'modules-tool': ("Modules tool to use", - 'choice', 'store', oldstyle_defaults['modules_tool'], - sorted(avail_modules_tools().keys())), + 'choice', 'store', DEFAULT_MODULES_TOOL, sorted(avail_modules_tools().keys())), 'prefix': (("Change prefix for buildpath, installpath, sourcepath and repositorypath " - "(used prefix for defaults %s)" % oldstyle_defaults['prefix']), + "(used prefix for defaults %s)" % DEFAULT_PREFIX), None, 'store', None), 'recursive-module-unload': ("Enable generating of modules that unload recursively.", None, 'store_true', False), 'repository': ("Repository type, using repositorypath", - 'choice', 'store', oldstyle_defaults['repository'], sorted(avail_repositories().keys())), + 'choice', 'store', DEFAULT_REPOSITORY, sorted(avail_repositories().keys())), 'repositorypath': (("Repository path, used by repository " "(is passed as list of arguments to create the repository instance). " "For more info, use --avail-repositories."), 'strlist', 'store', - oldstyle_defaults['repositorypath'][oldstyle_defaults['repository']]), + [mk_full_default_path('repositorypath')]), 'show-default-moduleclasses': ("Show default module classes with description", None, 'store_true', False), 'sourcepath': ("Path(s) to where sources should be downloaded (string, colon-separated)", - None, 'store', oldstyle_defaults['sourcepath']), - 'subdir-modules': ("Installpath subdir for modules", None, 'store', oldstyle_defaults['subdir_modules']), - 'subdir-software': ("Installpath subdir for software", None, 'store', oldstyle_defaults['subdir_software']), + None, 'store', mk_full_default_path('sourcepath')), + 'subdir-modules': ("Installpath subdir for modules", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_modules']), + 'subdir-software': ("Installpath subdir for software", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_software']), 'suffix-modules-path': ("Suffix for module files install path", None, 'store', GENERAL_CLASS), # this one is sort of an exception, it's something jobscripts can set, # has no real meaning for regular eb usage - 'testoutput': ("Path to where a job should place the output (to be set within jobscript)", - None, 'store', None), - 'tmp-logdir': ("Log directory where temporary log files are stored", - None, 'store', oldstyle_defaults['tmp_logdir']), + 'testoutput': ("Path to where a job should place the output (to be set within jobscript)", None, 'store', None), + 'tmp-logdir': ("Log directory where temporary log files are stored", None, 'store', None), 'tmpdir': ('Directory to use for temporary storage', None, 'store', None), }) @@ -252,13 +267,15 @@ def informative_options(self): descr = ("Informative options", "Obtain information about EasyBuild.") opts = OrderedDict({ + 'avail-cfgfile-constants': ("Show all constants that can be used in configuration files", + None, 'store_true', False), 'avail-easyconfig-constants': ("Show all constants that can be used in easyconfigs", None, 'store_true', False), 'avail-easyconfig-licenses': ("Show all license constants that can be used in easyconfigs", None, 'store_true', False), 'avail-easyconfig-params': (("Show all easyconfig parameters (include " "easyblock-specific ones by using -e)"), - None, "store_true", False, 'a'), + 'choice', 'store_or_None', FORMAT_TXT, [FORMAT_RST, FORMAT_TXT], 'a'), 'avail-easyconfig-templates': (("Show all template names and template constants " "that can be used in easyconfigs"), None, 'store_true', False), @@ -365,7 +382,7 @@ def postprocess(self): # prepare for --list/--avail if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates, - self.options.list_easyblocks, self.options.list_toolchains, + self.options.list_easyblocks, self.options.list_toolchains, self.options.avail_cfgfile_constants, self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses, self.options.avail_repositories, self.options.show_default_moduleclasses, self.options.avail_modules_tools, self.options.avail_module_naming_schemes, @@ -393,37 +410,39 @@ def postprocess(self): def _postprocess_config(self): """Postprocessing of configuration options""" if self.options.prefix is not None: - changed_defaults = get_default_oldstyle_configfile_defaults(self.options.prefix) # prefix applies to all paths, and repository has to be reinitialised to take new repositorypath into account # in the legacy-style configuration, repository is initialised in configuration file itself for dest in ['installpath', 'buildpath', 'sourcepath', 'repository', 'repositorypath']: if not self.options._action_taken.get(dest, False): - new_def = changed_defaults[dest] - if dest == 'repositorypath': - setattr(self.options, dest, new_def[changed_defaults['repository']]) + if dest == 'repository': + setattr(self.options, dest, DEFAULT_REPOSITORY) + elif dest == 'repositorypath': + setattr(self.options, dest, [mk_full_default_path(dest, prefix=self.options.prefix)]) else: - setattr(self.options, dest, new_def) - # LEGACY this line is here for oldstyle reasons - self.log.deprecated('Fake action taken to distinguish from default', '2.0') + setattr(self.options, dest, mk_full_default_path(dest, prefix=self.options.prefix)) + # LEGACY this line is here for oldstyle config reasons self.options._action_taken[dest] = True if self.options.pretend: self.options.installpath = get_pretend_installpath() - # split supplied list of robot paths to obtain a list - if self.options.robot: - class RobotPath(ListOfStrings): - SEPARATOR_LIST = os.pathsep - # explicit definition of __str__ is required for unknown reason related to the way Wrapper is defined - __str__ = ListOfStrings.__str__ - self.options.robot = RobotPath(self.options.robot) + if self.options.robot is not None: + # paths specified to --robot have preference over --robot-paths + # keep both values in sync if robot is enabled, which implies enabling dependency resolver + self.options.robot_paths = self.options.robot + self.options.robot_paths + self.options.robot = self.options.robot_paths def _postprocess_list_avail(self): """Create all the additional info that can be requested (exit at the end)""" msg = '' + + # dump supported configuration file constants + if self.options.avail_cfgfile_constants: + msg += self.avail_cfgfile_constants() + # dump possible easyconfig params if self.options.avail_easyconfig_params: - msg += self.avail_easyconfig_params() + msg += avail_easyconfig_params(self.options.easyblock, self.options.avail_easyconfig_params) # dump easyconfig template options if self.options.avail_easyconfig_templates: @@ -467,34 +486,22 @@ def _postprocess_list_avail(self): print msg sys.exit(0) - def avail_easyconfig_params(self): + def avail_cfgfile_constants(self): """ - Print the available easyconfig parameters, for the given easyblock. + Return overview of constants supported in configuration files. """ - app = get_easyblock_class(self.options.easyblock) - extra = app.extra_options() - mapping = convert_to_help(extra, has_default=False) - if len(extra) > 0: - ebb_msg = " (* indicates specific for the %s EasyBlock)" % app.__name__ - extra_names = [x[0] for x in extra] - else: - ebb_msg = '' - extra_names = [] - txt = ["Available easyconfig parameters%s" % ebb_msg] - params = [(k, v) for (k, v) in mapping.items() if k.upper() not in ['HIDDEN']] - for key, values in params: - txt.append("%s" % key.upper()) - txt.append('-' * len(key)) - for name, value in values: - tabs = "\t" * (3 - (len(name) + 1) / 8) - if name in extra_names: - starred = '(*)' - else: - starred = '' - txt.append("%s%s:%s%s" % (name, starred, tabs, value)) - txt.append('') - - return "\n".join(txt) + lines = [ + "Constants available (only) in configuration files:", + "syntax: %(CONSTANT_NAME)s", + ] + for section in self.go_cfg_constants: + lines.append('') + if section != self.DEFAULTSECT: + section_title = "only in '%s' section:" % section + lines.append(section_title) + for cst_name, (cst_value, cst_help) in sorted(self.go_cfg_constants[section].items()): + lines.append("* %s: %s [value: %s]" % (cst_name, cst_help, cst_value)) + return '\n'.join(lines) def avail_classes_tree(self, classes, classNames, detailed, depth=0): """Print list of classes as a tree.""" @@ -579,7 +586,7 @@ def avail_toolchains(self): def avail_repositories(self): """Show list of known repository types.""" - repopath_defaults = get_default_oldstyle_configfile_defaults()['repositorypath'] + repopath_defaults = mk_full_default_path('repositorypath') all_repos = avail_repositories(check_useable=False) usable_repos = avail_repositories(check_useable=True).keys() diff --git a/easybuild/tools/parallelbuild.py b/easybuild/tools/parallelbuild.py index 92967dae82..f71de69a5f 100644 --- a/easybuild/tools/parallelbuild.py +++ b/easybuild/tools/parallelbuild.py @@ -34,6 +34,7 @@ """ import math import os +import subprocess import easybuild.tools.config as config from easybuild.framework.easyblock import get_easyblock_instance @@ -118,12 +119,42 @@ def tokey(dep): return jobs +def submit_jobs(ordered_ecs, cmd_line_opts, testing=False): + """ + Submit jobs. + @param ordered_ecs: list of easyconfigs, in the order they should be processed + @param cmd_line_opts: list of command line options (in 'longopt=value' form) + """ + curdir = os.getcwd() + + # the options to ignore (help options can't reach here) + ignore_opts = ['robot', 'job'] + + # generate_cmd_line returns the options in form --longopt=value + opts = [x for x in cmd_line_opts if not x.split('=')[0] in ['--%s' % y for y in ignore_opts]] + + # compose string with command line options, properly quoted and with '%' characters escaped + opts_str = subprocess.list2cmdline(opts).replace('%', '%%') + + command = "unset TMPDIR && cd %s && eb %%(spec)s %s --testoutput=%%(output_dir)s" % (curdir, opts_str) + _log.info("Command template for jobs: %s" % command) + job_info_lines = [] + if testing: + _log.debug("Skipping actual submission of jobs since testing mode is enabled") + else: + jobs = build_easyconfigs_in_parallel(command, ordered_ecs) + job_info_lines = ["List of submitted jobs:"] + job_info_lines.extend(["%s (%s): %s" % (job.name, job.module, job.jobid) for job in jobs]) + job_info_lines.append("(%d jobs submitted)" % len(jobs)) + return '\n'.join(job_info_lines) + + def create_job(build_command, easyconfig, output_dir=None, conn=None, ppn=None): """ Creates a job, to build a *single* easyconfig @param build_command: format string for command, full path to an easyconfig file will be substituted in it @param easyconfig: easyconfig as processed by process_easyconfig - @param output_dir: optional output path; $EASYBUILDTESTOUTPUT will be set inside the job with this variable + @param output_dir: optional output path; --regtest-output-dir will be used inside the job with this variable @param conn: open connection to PBS server @param ppn: ppn setting to use (# 'processors' (cores) per node to use) returns the job @@ -131,9 +162,6 @@ def create_job(build_command, easyconfig, output_dir=None, conn=None, ppn=None): if output_dir is None: output_dir = 'easybuild-build' - # create command based on build_command template - command = build_command % {'spec': easyconfig['spec']} - # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD easybuild_vars = {} for name in os.environ: @@ -152,8 +180,11 @@ def create_job(build_command, easyconfig, output_dir=None, conn=None, ppn=None): ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec'])) name = '-'.join(ec_tuple) - var = config.OLDSTYLE_ENVIRONMENT_VARIABLES['test_output_path'] - easybuild_vars[var] = os.path.join(os.path.abspath(output_dir), name) + # create command based on build_command template + command = build_command % { + 'spec': easyconfig['spec'], + 'output_dir': os.path.join(os.path.abspath(output_dir), name), + } # just use latest build stats repo = init_repository(get_repository(), get_repositorypath()) diff --git a/easybuild/tools/pbs_job.py b/easybuild/tools/pbs_job.py index 930c059af9..55774cd474 100644 --- a/easybuild/tools/pbs_job.py +++ b/easybuild/tools/pbs_job.py @@ -150,11 +150,8 @@ def __init__(self, script, name, env_vars=None, resources={}, conn=None, ppn=Non "walltime": "%s:00:00" % hours, "nodes": "1:ppn=%s" % cores } - # set queue based on the hours requested - if hours >= 12: - self.queue = 'long' - else: - self.queue = 'short' + # don't specify any queue name to submit to, use the default + self.queue = None # job id of this job self.jobid = None # list of dependencies for this job diff --git a/easybuild/tools/repository/filerepo.py b/easybuild/tools/repository/filerepo.py index 7e1b2f792e..4c1a58fe17 100644 --- a/easybuild/tools/repository/filerepo.py +++ b/easybuild/tools/repository/filerepo.py @@ -34,7 +34,7 @@ @author: Jens Timmerman (Ghent University) @author: Toon Willems (Ghent University) @author: Ward Poelmans (Ghent University) -@author: Fotis Georgatos (University of Luxembourg) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ import os import time diff --git a/easybuild/tools/repository/gitrepo.py b/easybuild/tools/repository/gitrepo.py index f540e5ee1a..ab6a5f07ca 100644 --- a/easybuild/tools/repository/gitrepo.py +++ b/easybuild/tools/repository/gitrepo.py @@ -34,7 +34,7 @@ @author: Jens Timmerman (Ghent University) @author: Toon Willems (Ghent University) @author: Ward Poelmans (Ghent University) -@author: Fotis Georgatos (University of Luxembourg) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ import getpass import os diff --git a/easybuild/tools/repository/repository.py b/easybuild/tools/repository/repository.py index 3775bacfba..a8dca02326 100644 --- a/easybuild/tools/repository/repository.py +++ b/easybuild/tools/repository/repository.py @@ -32,7 +32,7 @@ @author: Jens Timmerman (Ghent University) @author: Toon Willems (Ghent University) @author: Ward Poelmans (Ghent University) -@author: Fotis Georgatos (University of Luxembourg) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ from vsc.utils import fancylogger from vsc.utils.missing import get_subclasses diff --git a/easybuild/tools/repository/svnrepo.py b/easybuild/tools/repository/svnrepo.py index cadb0e633b..31f38abdc9 100644 --- a/easybuild/tools/repository/svnrepo.py +++ b/easybuild/tools/repository/svnrepo.py @@ -34,7 +34,7 @@ @author: Jens Timmerman (Ghent University) @author: Toon Willems (Ghent University) @author: Ward Poelmans (Ghent University) -@author: Fotis Georgatos (University of Luxembourg) +@author: Fotis Georgatos (Uni.Lu, NTUA) """ import getpass import os diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py new file mode 100644 index 0000000000..99a3f6b9fa --- /dev/null +++ b/easybuild/tools/robot.py @@ -0,0 +1,247 @@ +# # +# Copyright 2009-2014 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # +""" +Dependency resolution functionality, a.k.a. robot. + +@author: Stijn De Weirdt (Ghent University) +@author: Dries Verdegem (Ghent University) +@author: Kenneth Hoste (Ghent University) +@author: Pieter De Baets (Ghent University) +@author: Jens Timmerman (Ghent University) +@author: Toon Willems (Ghent University) +@author: Ward Poelmans (Ghent University) +""" +import os +from vsc.utils import fancylogger + +from easybuild.framework.easyconfig.easyconfig import ActiveMNS, process_easyconfig, robot_find_easyconfig +from easybuild.framework.easyconfig.tools import find_resolved_modules, skip_available +from easybuild.tools.config import build_option +from easybuild.tools.filetools import det_common_path_prefix, search_file +from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS +from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version +from easybuild.tools.modules import modules_tool + + +_log = fancylogger.getLogger('tools.robot', fname=False) + + +def det_robot_path(robot_paths_option, tweaked_ecs_path, pr_path, auto_robot=False): + """Determine robot path.""" + robot_path = robot_paths_option[:] + _log.info("Using robot path(s): %s" % robot_path) + + # paths to tweaked easyconfigs or easyconfigs downloaded from a PR have priority + if tweaked_ecs_path is not None: + robot_path.insert(0, tweaked_ecs_path) + _log.info("Prepended list of robot search paths with %s: %s" % (tweaked_ecs_path, robot_path)) + if pr_path is not None: + robot_path.insert(0, pr_path) + _log.info("Prepended list of robot search paths with %s: %s" % (pr_path, robot_path)) + + return robot_path + + +def dry_run(easyconfigs, short=False, build_specs=None): + """ + Compose dry run overview for supplied easyconfigs ([ ] for unavailable, [x] for available, [F] for forced) + @param easyconfigs: list of parsed easyconfigs (EasyConfig instances) + @param short: use short format for overview: use a variable for common prefixes + @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) + """ + lines = [] + if build_option('robot_path') is None: + lines.append("Dry run: printing build status of easyconfigs") + all_specs = easyconfigs + else: + lines.append("Dry run: printing build status of easyconfigs and dependencies") + all_specs = resolve_dependencies(easyconfigs, build_specs=build_specs, retain_all_deps=True) + + unbuilt_specs = skip_available(all_specs) + dry_run_fmt = " * [%1s] %s (module: %s)" # markdown compatible (list of items with checkboxes in front) + + listed_ec_paths = [spec['spec'] for spec in easyconfigs] + + var_name = 'CFGS' + common_prefix = det_common_path_prefix([spec['spec'] for spec in all_specs]) + # only allow short if common prefix is long enough + short = short and common_prefix is not None and len(common_prefix) > len(var_name) * 2 + for spec in all_specs: + if spec in unbuilt_specs: + ans = ' ' + elif build_option('force') and spec['spec'] in listed_ec_paths: + ans = 'F' + else: + ans = 'x' + + if spec['ec'].short_mod_name != spec['ec'].full_mod_name: + mod = "%s | %s" % (spec['ec'].mod_subdir, spec['ec'].short_mod_name) + else: + mod = spec['ec'].full_mod_name + + if short: + item = os.path.join('$%s' % var_name, spec['spec'][len(common_prefix) + 1:]) + else: + item = spec['spec'] + lines.append(dry_run_fmt % (ans, item, mod)) + + if short: + # insert after 'Dry run:' message + lines.insert(1, "%s=%s" % (var_name, common_prefix)) + return '\n'.join(lines) + + +def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False): + """ + Work through the list of easyconfigs to determine an optimal order + @param unprocessed: list of easyconfigs + @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) + @param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability; + retain all deps when True, check matching build option when False + """ + + robot = build_option('robot_path') + # retain all dependencies if specified by either the resp. build option or the dedicated named argument + retain_all_deps = build_option('retain_all_deps') or retain_all_deps + + if retain_all_deps: + # assume that no modules are available when forced, to retain all dependencies + avail_modules = [] + _log.info("Forcing all dependencies to be retained.") + else: + # Get a list of all available modules (format: [(name, installversion), ...]) + avail_modules = modules_tool().available() + + if len(avail_modules) == 0: + _log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH')) + + ordered_ecs = [] + # all available modules can be used for resolving dependencies except those that will be installed + being_installed = [p['full_mod_name'] for p in unprocessed] + avail_modules = [m for m in avail_modules if not m in being_installed] + + _log.debug('unprocessed before resolving deps: %s' % unprocessed) + + # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though) + irresolvable = [] + loopcnt = 0 + maxloopcnt = 10000 + while unprocessed: + # make sure this stops, we really don't want to get stuck in an infinite loop + loopcnt += 1 + if loopcnt > maxloopcnt: + tup = (maxloopcnt, unprocessed, irresolvable) + msg = "Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)" % tup + _log.error(msg) + + # first try resolving dependencies without using external dependencies + last_processed_count = -1 + while len(avail_modules) > last_processed_count: + last_processed_count = len(avail_modules) + res = find_resolved_modules(unprocessed, avail_modules, retain_all_deps=retain_all_deps) + more_ecs, unprocessed, avail_modules = res + for ec in more_ecs: + if not ec['full_mod_name'] in [x['full_mod_name'] for x in ordered_ecs]: + ordered_ecs.append(ec) + + # robot: look for existing dependencies, add them + if robot and unprocessed: + + # rely on EasyBuild module naming scheme when resolving dependencies, since we know that will + # generate sensible module names that include the necessary information for the resolution to work + # (name, version, toolchain, versionsuffix) + being_installed = [EasyBuildMNS().det_full_module_name(p['ec']) for p in unprocessed] + + additional = [] + for entry in unprocessed: + # do not choose an entry that is being installed in the current run + # if they depend, you probably want to rebuild them using the new dependency + deps = entry['dependencies'] + candidates = [d for d in deps if not EasyBuildMNS().det_full_module_name(d) in being_installed] + if candidates: + cand_dep = candidates[0] + # find easyconfig, might not find any + _log.debug("Looking for easyconfig for %s" % str(cand_dep)) + # note: robot_find_easyconfig may return None + path = robot_find_easyconfig(cand_dep['name'], det_full_ec_version(cand_dep)) + + if path is None: + # no easyconfig found for dependency, add to list of irresolvable dependencies + if cand_dep not in irresolvable: + _log.debug("Irresolvable dependency found: %s" % cand_dep) + irresolvable.append(cand_dep) + # remove irresolvable dependency from list of dependencies so we can continue + entry['dependencies'].remove(cand_dep) + else: + _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) + # build specs should not be passed down to resolved dependencies, + # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself + hidden = cand_dep.get('hidden', False) + processed_ecs = process_easyconfig(path, validate=not retain_all_deps, hidden=hidden) + + # ensure that selected easyconfig provides required dependency + mods = [spec['ec'].full_mod_name for spec in processed_ecs] + dep_mod_name = ActiveMNS().det_full_module_name(cand_dep) + if not dep_mod_name in mods: + tup = (path, dep_mod_name, mods) + _log.error("easyconfig file %s does not contain module %s (mods: %s)" % tup) + + for ec in processed_ecs: + if not ec in unprocessed + additional: + additional.append(ec) + _log.debug("Added %s as dependency of %s" % (ec, entry)) + else: + mod_name = EasyBuildMNS().det_full_module_name(entry['ec']) + _log.debug("No more candidate dependencies to resolve for %s" % mod_name) + + # add additional (new) easyconfigs to list of stuff to process + unprocessed.extend(additional) + + elif not robot: + # no use in continuing if robot is not enabled, dependencies won't be resolved anyway + irresolvable = [dep for x in unprocessed for dep in x['dependencies']] + break + + if irresolvable: + _log.warning("Irresolvable dependencies (details): %s" % irresolvable) + irresolvable_mods_eb = [EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable] + _log.warning("Irresolvable dependencies (EasyBuild module names): %s" % ', '.join(irresolvable_mods_eb)) + irresolvable_mods = [ActiveMNS().det_full_module_name(dep) for dep in irresolvable] + _log.error('Irresolvable dependencies encountered: %s' % ', '.join(irresolvable_mods)) + + _log.info("Dependency resolution complete, building as follows:\n%s" % ordered_ecs) + return ordered_ecs + + +def search_easyconfigs(query, short=False): + """Search for easyconfigs, if a query is provided.""" + robot_path = build_option('robot_path') + if robot_path: + search_path = robot_path + else: + search_path = [os.getcwd()] + ignore_dirs = build_option('ignore_dirs') + silent = build_option('silent') + search_file(search_path, query, short=short, ignore_dirs=ignore_dirs, silent=silent) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 9b42b159fe..ba13928464 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -376,18 +376,17 @@ def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp): if not regexp: use_regexp = False + _log.debug('cmd "%s" exited with exitcode %s and output:\n%s' % (cmd, ec, stdouterr)) + if ec and (log_all or log_ok): # We don't want to error if the user doesn't care if check_ec: _log.error('cmd "%s" exited with exitcode %s and output:\n%s' % (cmd, ec, stdouterr)) else: _log.warn('cmd "%s" exited with exitcode %s and output:\n%s' % (cmd, ec, stdouterr)) - - if not ec: + elif not ec: if log_all: _log.info('cmd "%s" exited with exitcode %s and output:\n%s' % (cmd, ec, stdouterr)) - else: - _log.debug('cmd "%s" exited with exitcode %s and output:\n%s' % (cmd, ec, stdouterr)) # parse the stdout/stderr for errors when strictness dictates this or when regexp is passed in if use_regexp or regexp: diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 7cd2babc04..2d2fbc696c 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -128,12 +128,8 @@ def count_bits(n): def get_core_count(): - """ - Try to detect the number of virtual or physical CPUs on this system - (DEPRECATED, use get_avail_core_count instead) - """ - _log.deprecated("get_core_count() is deprecated, use get_avail_core_count() instead", '2.0') - return get_avail_core_count() + """NO LONGER SUPPORTED: use get_avail_core_count() instead""" + _log.nosupport("get_core_count() is replaced by get_avail_core_count()", '2.0') def get_cpu_vendor(): @@ -259,16 +255,8 @@ def get_cpu_speed(): def get_kernel_name(): - """Try to determine kernel name - - e.g., 'Linux', 'Darwin', ... - """ - _log.deprecated("get_kernel_name() (replaced by get_os_type())", "2.0") - try: - kernel_name = os.uname()[0] - return kernel_name - except OSError, err: - raise SystemToolsException("Failed to determine kernel name: %s" % err) + """NO LONGER SUPPORTED: use get_os_type() instead""" + _log.nosupport("get_kernel_name() is replaced by get_os_type()", '2.0') def get_os_type(): @@ -326,15 +314,9 @@ def get_os_name(): Determine system name, e.g., 'redhat' (generic), 'centos', 'debian', 'fedora', 'suse', 'ubuntu', 'red hat enterprise linux server', 'SL' (Scientific Linux), 'opensuse', ... """ - try: - # platform.linux_distribution is more useful, but only available since Python 2.6 - # this allows to differentiate between Fedora, CentOS, RHEL and Scientific Linux (Rocks is just CentOS) - os_name = platform.linux_distribution()[0].strip().lower() - except AttributeError: - # platform.dist can be used as a fallback - # CentOS, RHEL, Rocks and Scientific Linux may all appear as 'redhat' (especially if Python version is pre v2.6) - os_name = platform.dist()[0].strip().lower() - _log.deprecated("platform.dist as fallback for platform.linux_distribution", "2.0") + # platform.linux_distribution is more useful, but only available since Python 2.6 + # this allows to differentiate between Fedora, CentOS, RHEL and Scientific Linux (Rocks is just CentOS) + os_name = platform.linux_distribution()[0].strip().lower() os_name_map = { 'red hat enterprise linux server': 'RHEL', @@ -391,27 +373,24 @@ def check_os_dependency(dep): # - uses rpm -q and dpkg -s --> can be run as non-root!! # - fallback on which # - should be extended to files later? + found = None cmd = None - if get_os_name() in ['debian', 'ubuntu']: - if which('dpkg'): - cmd = "dpkg -s %s" % dep - else: - # OK for get_os_name() == redhat, fedora, RHEL, SL, centos - if which('rpm'): - cmd = "rpm -q %s" % dep + if which('rpm'): + cmd = "rpm -q %s" % dep + found = run_cmd(cmd, simple=True, log_all=False, log_ok=False) - found = None - if cmd is not None: + if not found and which('dpkg'): + cmd = "dpkg -s %s" % dep found = run_cmd(cmd, simple=True, log_all=False, log_ok=False) - if found is None: + if cmd is None: # fallback for when os-dependency is a binary/library found = which(dep) - # try locate if it's available - if found is None and which('locate'): - cmd = 'locate --regexp "/%s$"' % dep - found = run_cmd(cmd, simple=True, log_all=False, log_ok=False) + # try locate if it's available + if not found and which('locate'): + cmd = 'locate --regexp "/%s$"' % dep + found = run_cmd(cmd, simple=True, log_all=False, log_ok=False) return found @@ -464,7 +443,6 @@ def get_system_info(): 'gcc_version': get_tool_version('gcc', version_option='-v'), 'hostname': gethostname(), 'glibc_version': get_glibc_version(), - 'kernel_name': get_kernel_name(), 'os_name': get_os_name(), 'os_type': get_os_type(), 'os_version': get_os_version(), diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index b572921f7e..8bebbf3612 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -42,15 +42,16 @@ import easybuild.tools.config as config from easybuild.framework.easyblock import build_easyconfigs -from easybuild.framework.easyconfig.tools import process_easyconfig, resolve_dependencies +from easybuild.framework.easyconfig.tools import process_easyconfig from easybuild.framework.easyconfig.tools import skip_available from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option -from easybuild.tools.filetools import find_easyconfigs, mkdir, read_file +from easybuild.tools.filetools import find_easyconfigs, mkdir, read_file, write_file from easybuild.tools.github import create_gist, post_comment_in_issue from easybuild.tools.jenkins import aggregate_xml_in_dirs from easybuild.tools.modules import modules_tool from easybuild.tools.parallelbuild import build_easyconfigs_in_parallel +from easybuild.tools.robot import resolve_dependencies from easybuild.tools.systemtools import get_system_info from easybuild.tools.version import FRAMEWORK_VERSION, EASYBLOCKS_VERSION from vsc.utils import fancylogger @@ -75,19 +76,17 @@ def regtest(easyconfig_paths, build_specs=None): _log.info("aggregated xml files inside %s, output written to: %s" % (aggregate_regtest, output_file)) sys.exit(0) - # create base directory, which is used to place - # all log files and the test output as xml - basename = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S") - var = config.OLDSTYLE_ENVIRONMENT_VARIABLES['test_output_path'] - + # create base directory, which is used to place all log files and the test output as xml regtest_output_dir = build_option('regtest_output_dir') + testoutput = build_option('testoutput') if regtest_output_dir is not None: output_dir = regtest_output_dir - elif var in os.environ: - output_dir = os.path.abspath(os.environ[var]) + elif testoutput is not None: + output_dir = os.path.abspath(testoutput) else: # default: current dir + easybuild-test-[timestamp] - output_dir = os.path.join(cur_dir, basename) + dirname = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S") + output_dir = os.path.join(cur_dir, dirname) mkdir(output_dir, parents=True) @@ -120,7 +119,7 @@ def regtest(easyconfig_paths, build_specs=None): else: resolved = resolve_dependencies(easyconfigs, build_specs=build_specs) - cmd = "eb %(spec)s --regtest --sequential -ld" + cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s" command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd) # retry twice in case of failure, to avoid fluke errors command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {'cmd': cmd} @@ -157,9 +156,9 @@ def session_state(): } -def session_module_list(): +def session_module_list(testing=False): """Get list of loaded modules ('module list').""" - modtool = modules_tool() + modtool = modules_tool(testing=testing) return modtool.list() @@ -185,7 +184,7 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_l build_overview = [] for (ec, ec_res) in ecs_with_res: test_log = '' - if ec_res['success']: + if ec_res.get('success', False): test_result = 'SUCCESS' else: # compose test result string @@ -299,3 +298,38 @@ def post_easyconfigs_pr_test_report(pr_nr, test_report, msg, init_session_state, msg = "Test report uploaded to %s and mentioned in a comment in easyconfigs PR#%s" % (gist_url, pr_nr) return msg + + +def overall_test_report(ecs_with_res, orig_cnt, success, msg, init_session_state): + """ + Upload/dump overall test report + @param ecs_with_res: processed easyconfigs with build result (success/failure) + @param orig_cnt: number of original easyconfig paths + @param success: boolean indicating whether all builds were successful + @param msg: message to be included in test report + @param init_session_state: initial session state info to include in test report + """ + dump_path = build_option('dump_test_report') + pr_nr = build_option('from_pr') + upload = build_option('upload_test_report') + + if upload: + msg = msg + " (%d easyconfigs in this PR)" % orig_cnt + test_report = create_test_report(msg, ecs_with_res, init_session_state, pr_nr=pr_nr, gist_log=True) + if pr_nr: + # upload test report to gist and issue a comment in the PR to notify + txt = post_easyconfigs_pr_test_report(pr_nr, test_report, msg, init_session_state, success) + else: + # only upload test report as a gist + gist_url = upload_test_report_as_gist(test_report) + txt = "Test report uploaded to %s" % gist_url + else: + test_report = create_test_report(msg, ecs_with_res, init_session_state) + txt = None + _log.debug("Test report: %s" % test_report) + + if dump_path is not None: + write_file(dump_path, test_report) + _log.info("Test report dumped to %s" % dump_path) + + return txt diff --git a/easybuild/tools/toolchain/mpi.py b/easybuild/tools/toolchain/mpi.py index a04bb710ef..7a9694de8c 100644 --- a/easybuild/tools/toolchain/mpi.py +++ b/easybuild/tools/toolchain/mpi.py @@ -28,8 +28,8 @@ @author: Stijn De Weirdt (Ghent University) @author: Kenneth Hoste (Ghent University) """ - import os +import tempfile import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain @@ -167,15 +167,20 @@ def mpi_cmd_for(self, cmd, nr_ranks): """Construct an MPI command for the given command and number of ranks.""" # parameter values for mpirun command - params = {'nr_ranks':nr_ranks, 'cmd':cmd} + params = { + 'nr_ranks': nr_ranks, + 'cmd': cmd, + } # different known mpirun commands + mpirun_n_cmd = "mpirun -n %(nr_ranks)d %(cmd)s" mpi_cmds = { - toolchain.OPENMPI: "mpirun -n %(nr_ranks)d %(cmd)s", # @UndefinedVariable + toolchain.OPENMPI: mpirun_n_cmd, # @UndefinedVariable toolchain.QLOGICMPI: "mpirun -H localhost -np %(nr_ranks)d %(cmd)s", # @UndefinedVariable toolchain.INTELMPI: "mpirun %(mpdbf)s %(nodesfile)s -np %(nr_ranks)d %(cmd)s", # @UndefinedVariable - toolchain.MVAPICH2: "mpirun -n %(nr_ranks)d %(cmd)s", # @UndefinedVariable - toolchain.MPICH2: "mpirun -n %(nr_ranks)d %(cmd)s", # @UndefinedVariable + toolchain.MVAPICH2: mpirun_n_cmd, # @UndefinedVariable + toolchain.MPICH: mpirun_n_cmd, # @UndefinedVariable + toolchain.MPICH2: mpirun_n_cmd, # @UndefinedVariable } mpi_family = self.mpi_family() @@ -183,8 +188,10 @@ def mpi_cmd_for(self, cmd, nr_ranks): # Intel MPI mpirun needs more work if mpi_family == toolchain.INTELMPI: # @UndefinedVariable + tmpdir = tempfile.mkdtemp(prefix='eb-mpi_cmd_for-') + # set temporary dir for mdp - env.setvar('I_MPI_MPD_TMPDIR', "/tmp") + env.setvar('I_MPI_MPD_TMPDIR', tmpdir) # set PBS_ENVIRONMENT, so that --file option for mpdboot isn't stripped away env.setvar('PBS_ENVIRONMENT', "PBS_BATCH_MPI") @@ -194,7 +201,7 @@ def mpi_cmd_for(self, cmd, nr_ranks): env.setvar('I_MPI_PROCESS_MANAGER', 'mpd') # create mpdboot file - fn = "/tmp/mpdboot" + fn = os.path.join(tmpdir, 'mpdboot') try: if os.path.exists(fn): os.remove(fn) @@ -202,10 +209,10 @@ def mpi_cmd_for(self, cmd, nr_ranks): except OSError, err: self.log.error("Failed to create file %s: %s" % (fn, err)) - params.update({'mpdbf':"--file=%s" % fn}) + params.update({'mpdbf': "--file=%s" % fn}) # create nodes file - fn = "/tmp/nodes" + fn = os.path.join(tmpdir, 'nodes') try: if os.path.exists(fn): os.remove(fn) @@ -213,7 +220,7 @@ def mpi_cmd_for(self, cmd, nr_ranks): except OSError, err: self.log.error("Failed to create file %s: %s" % (fn, err)) - params.update({'nodesfile':"-machinefile %s" % fn}) + params.update({'nodesfile': "-machinefile %s" % fn}) if mpi_family in mpi_cmds.keys(): return mpi_cmds[mpi_family] % params diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 75c5a29c85..a4119c4592 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -34,7 +34,6 @@ import os import re from vsc.utils import fancylogger -from vsc.utils.missing import all, any from easybuild.tools.config import build_option, install_path from easybuild.tools.environment import setvar @@ -82,13 +81,13 @@ def __init__(self, name=None, version=None, mns=None): if name is None: name = self.NAME if name is None: - self.log.raiseException("init: no name provided") + self.log.error("Toolchain init: no name provided") self.name = name if version is None: version = self.VERSION if version is None: - self.log.raiseException("init: no version provided") + self.log.error("Toolchain init: no version provided") self.version = version self.vars = None @@ -129,7 +128,7 @@ def get_variable(self, name, typ=str): elif typ == list: return self.variables[name].flatten() else: - self.log.raiseException("get_variables: Don't know how to create value of type %s." % typ) + self.log.error("get_variable: Don't know how to create value of type %s." % typ) def set_variables(self): """Do nothing? Everything should have been set by others @@ -188,7 +187,7 @@ def _get_software_root(self, name): """Try to get the software root for name""" root = get_software_root(name) if root is None: - self.log.raiseException("get_software_root software root for %s was not found in environment" % (name)) + self.log.error("get_software_root software root for %s was not found in environment" % name) else: self.log.debug("get_software_root software root %s for %s was found in environment" % (root, name)) return root @@ -197,11 +196,9 @@ def _get_software_version(self, name): """Try to get the software root for name""" version = get_software_version(name) if version is None: - self.log.raiseException("get_software_version software version for %s was not found in environment" % - (name)) + self.log.error("get_software_version software version for %s was not found in environment" % name) else: - self.log.debug("get_software_version software version %s for %s was found in environment" % - (version, name)) + self.log.debug("get_software_version software version %s for %s was found in environment" % (version, name)) return version @@ -249,8 +246,8 @@ def set_options(self, options): self.options[opt] = options[opt] else: # used to be warning, but this is a severe error imho - self.log.raiseException("set_options: undefined toolchain option %s specified (possible names %s)" % - (opt, ",".join(self.options.keys()))) + known_opts = ','.join(self.options.keys()) + self.log.error("Undefined toolchain option %s specified (known options: %s)" % (opt, known_opts)) def get_dependency_version(self, dependency): """ Generate a version string for a dependency on a module using this toolchain """ @@ -281,8 +278,8 @@ def get_dependency_version(self, dependency): self.log.debug("get_dependency_version: version not in dependency return %s" % version) return else: - self.log.raiseException('get_dependency_version: No toolchain version for dependency '\ - 'name %s (suffix %s) found' % (dependency['name'], toolchain_suffix)) + tup = (dependency['name'], toolchain_suffix) + self.log.error("No toolchain version for dependency name %s (suffix %s) found" % tup) def add_dependencies(self, dependencies): """ Verify if the given dependencies exist and add them """ @@ -331,10 +328,10 @@ def prepare(self, onlymod=None): (If string: comma separated list of variables that will be ignored). """ if self.modules_tool is None: - self.log.raiseException("No modules tool defined.") + self.log.error("No modules tool defined in Toolchain instance.") if not self._toolchain_exists(): - self.log.raiseException("No module found for toolchain name '%s' (%s)" % (self.name, self.version)) + self.log.error("No module found for toolchain: %s" % self.mod_short_name) if self.name == DUMMY_TOOLCHAIN_NAME: if self.version == DUMMY_TOOLCHAIN_VERSION: @@ -429,7 +426,7 @@ def _setenv_variables(self, donotset=None): donotsetlist = [] if isinstance(donotset, str): # TODO : more legacy code that should be using proper type - self.log.raiseException("_setenv_variables: using commas-separated list. should be deprecated.") + self.log.error("_setenv_variables: using commas-separated list. should be deprecated.") donotsetlist = donotset.split(',') elif isinstance(donotset, list): donotsetlist = donotset @@ -459,37 +456,3 @@ def comp_family(self): def mpi_family(self): """ Return type of MPI library used in this toolchain (abstract method).""" raise NotImplementedError - - # legacy functions TODO remove AFTER migration - # should search'n'replaced - def get_type(self, name, type_map): - """Determine type of toolchain based on toolchain dependencies.""" - self.log.raiseException("get_type: legacy code. should not be needed anymore.") - - def _set_variables(self, dontset=None): - """ Sets the environment variables """ - self.log.raiseException("_set_variables: legacy code. use _setenv_variables.") - - def _addDependencyVariables(self, names=None): - """ Add LDFLAGS and CPPFLAGS to the self.vars based on the dependencies - names should be a list of strings containing the name of the dependency""" - self.log.raiseException("_addDependencyVaraibles: legacy code. use _add_dependency_variables.") - - def _setVariables(self, dontset=None): - """ Sets the environment variables """ - self.log.raiseException("_setVariables: legacy code. use _set_variables.") - - def _toolkitExists(self, name=None, version=None): - """ - Verify if there exists a toolkit by this name and version - """ - self.log.raiseException("_toolkitExists: legacy code. replace use _toolchain_exists.") - - def get_openmp_flag(self): - """Get compiler flag for OpenMP support.""" - self.log.raiseException("get_openmp_flag: legacy code. use options.get_flag('openmp').") - - @property - def opts(self): - """Get value for specified option.""" - self.log.raiseException("opts[x]: legacy code. use options[x].") diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index 246619e3fd..7c8a959319 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -32,8 +32,6 @@ import string import sys from vsc.utils import fancylogger -from vsc.utils.missing import any as _any -from vsc.utils.missing import all as _all import easybuild.tools.environment as env _log = fancylogger.getLogger('tools.utilities') @@ -45,26 +43,9 @@ UNWANTED_CHARS = ASCII_CHARS.translate(ASCII_CHARS, string.digits + string.ascii_letters + "_") -def any(ls): - """Reimplementation of 'any' function, which is not available in Python 2.4 yet.""" - _log.deprecated("own definition of any", "2.0") - return _any(ls) - - -def all(ls): - """Reimplementation of 'all' function, which is not available in Python 2.4 yet.""" - _log.deprecated("own definition of all", "2.0") - return _all(ls) - - def read_environment(env_vars, strict=False): - """ - Read variables from the environment - @param: env_vars: a dict with key a name, value a environment variable name - @param: strict, boolean, if True enforces that all specified environment variables are found - """ - _log.deprecated("moved read_environment to tools.environment", "2.0") - return env.read_environment(env_vars, strict) + """NO LONGER SUPPORTED: use read_environment from easybuild.tools.environment instead""" + _log.nosupport("read_environment has been moved to easybuild.tools.environment", '2.0') def flatten(lst): diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index f527eaecee..1bd1419af5 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -37,7 +37,7 @@ # note: release candidates should be versioned as a pre-release, e.g. "1.1rc1" # 1.1-rc1 would indicate a post-release, i.e., and update of 1.1, so beware! -VERSION = LooseVersion("1.16.0dev") +VERSION = LooseVersion("2.0.0dev") UNKNOWN = "UNKNOWN" def get_git_revision(): diff --git a/eb b/eb index 4bb70df087..51eeb7152a 100755 --- a/eb +++ b/eb @@ -39,6 +39,28 @@ # @author: Pieter De Baets (Ghent University) # @author: Jens Timmerman (Ghent University) +# Python 2.6 or more recent 2.x required +REQ_MAJ_PYVER=2 +REQ_MIN_PYVER=6 +REQ_PYVER=${REQ_MAJ_PYVER}.${REQ_MIN_PYVER} + +# make sure Python version being used is compatible +pyver=`python -V 2>&1 | cut -f2 -d' '` +pyver_maj=`echo $pyver | cut -f1 -d'.'` +pyver_min=`echo $pyver | cut -f2 -d'.'` + +if [ $pyver_maj -ne $REQ_MAJ_PYVER ] +then + echo "ERROR: EasyBuild is currently only compatible with Python v${REQ_MAJ_PYVER}.x, found v${pyver}" 1>&2 + exit 1 +fi +if [ $pyver_min -lt $REQ_MIN_PYVER ] +then + echo "ERROR: EasyBuild requires Python v${REQ_PYVER} or a more recent v${REQ_MAJ_PYVER}.x, found v${pyver}." 1>&2 + exit 2 +fi + + main_script_base_path="easybuild/main.py" python_search_path_cmd="python -c \"import sys; print ' '.join(sys.path)\"" diff --git a/setup.py b/setup.py index 5a707b3d82..1a961f4a36 100644 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ def find_rel_test(): easybuild_packages = [ "easybuild", "easybuild.framework", "easybuild.framework.easyconfig", "easybuild.framework.easyconfig.format", "easybuild.toolchains", "easybuild.toolchains.compiler", "easybuild.toolchains.mpi", - "easybuild.toolchains.fft", "easybuild.toolchains.linalg", "easybuild.tools", + "easybuild.toolchains.fft", "easybuild.toolchains.linalg", "easybuild.tools", "easybuild.tools.deprecated", "easybuild.tools.toolchain", "easybuild.tools.module_naming_scheme", "easybuild.tools.repository", "test.framework", "test", "vsc", "vsc.utils", @@ -82,9 +82,7 @@ def find_rel_test(): version = str(VERSION), author = "EasyBuild community", author_email = "easybuild@lists.ugent.be", - description = """EasyBuild is a software installation framework in Python that allows you to \ -install software in a structured and robust way. -This package contains the EasyBuild framework, which supports the creation of custom easyblocks that \ + description = """The EasyBuild framework supports the creation of custom easyblocks that \ implement support for installing particular (groups of) software packages.""", license = "GPLv2", keywords = "software build building installation installing compilation HPC scientific", @@ -93,15 +91,8 @@ def find_rel_test(): package_dir = {'test.framework': "test/framework"}, package_data = {"test.framework": find_rel_test()}, scripts = ["eb", "optcomplete.bash", "minimal_bash_completion.bash"], - data_files = [ - ('easybuild', ["easybuild/easybuild_config.py"]), - ], - long_description = """This package contains the EasyBuild -framework, which supports the creation of custom easyblocks that -implement support for installing particular (groups of) software -packages. - -""" + read("README.rst"), + data_files = [], + long_description = read('README.rst'), classifiers = [ "Development Status :: 5 - Production/Stable", "Environment :: Console", diff --git a/test/framework/config.py b/test/framework/config.py index b28ec141cf..21ee31185a 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -28,9 +28,9 @@ @author: Kenneth Hoste (Ghent University) @author: Stijn De Weirdt (Ghent University) """ -import copy import os import shutil +import sys import tempfile from test.framework.utilities import EnhancedTestCase, init_config from unittest import TestLoader @@ -38,11 +38,11 @@ from vsc.utils.fancylogger import setLogLevelDebug, logToScreen import easybuild.tools.options as eboptions -from easybuild.tools.config import build_path, source_paths, install_path, get_repository, get_repositorypath -from easybuild.tools.config import log_file_format, set_tmpdir, BuildOptions, ConfigurationVariables +from easybuild.tools.config import build_path, source_paths, install_path, get_repositorypath +from easybuild.tools.config import set_tmpdir, BuildOptions, ConfigurationVariables from easybuild.tools.config import get_build_log_path, DEFAULT_PATH_SUBDIRS, init_build_options, build_option from easybuild.tools.environment import modify_env -from easybuild.tools.filetools import write_file +from easybuild.tools.filetools import mkdir, write_file from easybuild.tools.repository.filerepo import FileRepository from easybuild.tools.repository.repository import init_repository @@ -105,254 +105,8 @@ def test_default_config(self): self.assertEqual(config_options['repositorypath'], [os.path.join(eb_homedir, 'ebfiles_repo')]) self.assertEqual(config_options['logfile_format'][0], 'easybuild') self.assertEqual(config_options['logfile_format'][1], "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log") - self.assertEqual(config_options['tmp_logdir'], tempfile.gettempdir()) - - def test_generaloption_overrides_legacy(self): - """Test whether generaloption overrides legacy configuration.""" - self.purge_environment() - # if both legacy and generaloption style configuration is mixed, generaloption wins - legacy_prefix = os.path.join(self.tmpdir, 'legacy') - go_prefix = os.path.join(self.tmpdir, 'generaloption') - - # legacy env vars - os.environ['EASYBUILDPREFIX'] = legacy_prefix - os.environ['EASYBUILDBUILDPATH'] = os.path.join(legacy_prefix, 'build') - # generaloption env vars - os.environ['EASYBUILD_INSTALLPATH'] = go_prefix - init_config() - self.assertEqual(build_path(), os.path.join(legacy_prefix, 'build')) - self.assertEqual(install_path(), os.path.join(go_prefix, 'software')) - repo = init_repository(get_repository(), get_repositorypath()) - self.assertEqual(repo.repo, os.path.join(legacy_prefix, 'ebfiles_repo')) - del os.environ['EASYBUILDPREFIX'] - - # legacy env vars - os.environ['EASYBUILDBUILDPATH'] = os.path.join(legacy_prefix, 'buildagain') - # generaloption env vars - os.environ['EASYBUILD_PREFIX'] = go_prefix - init_config() - self.assertEqual(build_path(), os.path.join(go_prefix, 'build')) - self.assertEqual(install_path(), os.path.join(go_prefix, 'software')) - repo = init_repository(get_repository(), get_repositorypath()) - self.assertEqual(repo.repo, os.path.join(go_prefix, 'ebfiles_repo')) - del os.environ['EASYBUILDBUILDPATH'] - - def test_legacy_env_vars(self): - """Test legacy environment variables.""" - self.purge_environment() - - # build path - test_buildpath = os.path.join(self.tmpdir, 'build', 'path') - os.environ['EASYBUILDBUILDPATH'] = test_buildpath - self.configure(args=[]) - self.assertEqual(build_path(), test_buildpath) - del os.environ['EASYBUILDBUILDPATH'] - - # source path(s) - test_sourcepaths = [ - os.path.join(self.tmpdir, 'source', 'path'), - ':'.join([ - os.path.join(self.tmpdir, 'source', 'path1'), - os.path.join(self.tmpdir, 'source', 'path2'), - ]), - ':'.join([ - os.path.join(self.tmpdir, 'source', 'path1'), - os.path.join(self.tmpdir, 'source', 'path2'), - os.path.join(self.tmpdir, 'source', 'path3'), - ]), - ] - for test_sourcepath in test_sourcepaths: - init_config() - os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath - self.configure(args=[]) - self.assertEqual(build_path(), os.path.join(os.path.expanduser('~'), '.local', 'easybuild', - DEFAULT_PATH_SUBDIRS['buildpath'])) - self.assertEqual(source_paths(), test_sourcepath.split(':')) - del os.environ['EASYBUILDSOURCEPATH'] - - test_sourcepath = os.path.join(self.tmpdir, 'source', 'path') - - # install path - init_config() - test_installpath = os.path.join(self.tmpdir, 'install', 'path') - os.environ['EASYBUILDINSTALLPATH'] = test_installpath - self.configure(args=[]) - self.assertEqual(source_paths()[0], os.path.join(os.path.expanduser('~'), '.local', 'easybuild', - DEFAULT_PATH_SUBDIRS['sourcepath'])) - self.assertEqual(install_path(), os.path.join(test_installpath, DEFAULT_PATH_SUBDIRS['subdir_software'])) - self.assertEqual(install_path(typ='mod'), os.path.join(test_installpath, - DEFAULT_PATH_SUBDIRS['subdir_modules'])) - del os.environ['EASYBUILDINSTALLPATH'] - - # prefix: should change build/install/source/repo paths - init_config() - test_prefixpath = os.path.join(self.tmpdir, 'prefix', 'path') - os.environ['EASYBUILDPREFIX'] = test_prefixpath - self.configure(args=[]) - self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath'])) - self.assertEqual(source_paths()[0], os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath'])) - self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software'])) - self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath, - DEFAULT_PATH_SUBDIRS['subdir_modules'])) - repo = init_repository(get_repository(), get_repositorypath()) - self.assertTrue(isinstance(repo, FileRepository)) - self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath'])) - # purposely not unsetting $EASYBUILDPREFIX yet here - - # build/source/install path overrides prefix - init_config() - os.environ['EASYBUILDBUILDPATH'] = test_buildpath - self.configure(args=[]) - self.assertEqual(build_path(), test_buildpath) - self.assertEqual(source_paths()[0], os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath'])) - self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software'])) - self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath, - DEFAULT_PATH_SUBDIRS['subdir_modules'])) - repo = init_repository(get_repository(), get_repositorypath()) - self.assertTrue(isinstance(repo, FileRepository)) - self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath'])) - del os.environ['EASYBUILDBUILDPATH'] - - init_config() - os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath - self.configure(args=[]) - self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath'])) - self.assertEqual(source_paths()[0], test_sourcepath) - self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software'])) - self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath, - DEFAULT_PATH_SUBDIRS['subdir_modules'])) - repo = init_repository(get_repository(), get_repositorypath()) - self.assertTrue(isinstance(repo, FileRepository)) - self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath'])) - del os.environ['EASYBUILDSOURCEPATH'] - - init_config() - os.environ['EASYBUILDINSTALLPATH'] = test_installpath - self.configure(args=[]) - self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath'])) - self.assertEqual(source_paths()[0], os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath'])) - self.assertEqual(install_path(), os.path.join(test_installpath, DEFAULT_PATH_SUBDIRS['subdir_software'])) - self.assertEqual(install_path(typ='mod'), os.path.join(test_installpath, - DEFAULT_PATH_SUBDIRS['subdir_modules'])) - repo = init_repository(get_repository(), get_repositorypath()) - self.assertTrue(isinstance(repo, FileRepository)) - self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath'])) - del os.environ['EASYBUILDPREFIX'] - del os.environ['EASYBUILDINSTALLPATH'] - - def test_legacy_config_file(self): - """Test finding/using legacy configuration files.""" - self.purge_environment() - - cfg_fn = self.configure(args=[]) - self.assertTrue(cfg_fn.endswith('easybuild/easybuild_config.py')) - - configtxt = """ -build_path = '%(buildpath)s' -source_path = '%(sourcepath)s' -install_path = '%(installpath)s' -repository_path = '%(repopath)s' -repository = FileRepository(repository_path) -log_format = ('%(logdir)s', '%(logtmpl)s') -log_dir = '%(tmplogdir)s' -software_install_suffix = '%(softsuffix)s' -modules_install_suffix = '%(modsuffix)s' -""" - - buildpath = os.path.join(self.tmpdir, 'my', 'test', 'build', 'path') - sourcepath = os.path.join(self.tmpdir, 'my', 'test', 'source', 'path') - installpath = os.path.join(self.tmpdir, 'my', 'test', 'install', 'path') - repopath = os.path.join(self.tmpdir, 'my', 'test', 'repo', 'path') - logdir = 'somedir' - logtmpl = 'test-eb-%(name)s%(version)s_date-%(date)s__time-%(time)s.log' - tmplogdir = os.path.join(self.tmpdir, 'my', 'test', 'tmplogdir') - softsuffix = 'myfavoritesoftware' - modsuffix = 'modulesgohere' - - configdict = { - 'buildpath': buildpath, - 'sourcepath': sourcepath, - 'installpath': installpath, - 'repopath': repopath, - 'logdir': logdir, - 'logtmpl': logtmpl, - 'tmplogdir': tmplogdir, - 'softsuffix': softsuffix, - 'modsuffix': modsuffix - } - - # create user config file on default location - myconfigfile = os.path.join(self.tmpdir, '.easybuild', 'config.py') - if not os.path.exists(os.path.dirname(myconfigfile)): - os.makedirs(os.path.dirname(myconfigfile)) - write_file(myconfigfile, configtxt % configdict) - - # redefine home so we can test user config file on default location - home = os.environ.get('HOME', None) - os.environ['HOME'] = self.tmpdir - init_config() - cfg_fn = self.configure(args=[]) - if home is not None: - os.environ['HOME'] = home - - # check finding and use of config file - self.assertEqual(cfg_fn, myconfigfile) - self.assertEqual(build_path(), buildpath) - self.assertEqual(source_paths()[0], sourcepath) - self.assertEqual(install_path(), os.path.join(installpath, softsuffix)) - self.assertEqual(install_path(typ='mod'), os.path.join(installpath, modsuffix)) - repo = init_repository(get_repository(), get_repositorypath()) - self.assertTrue(isinstance(repo, FileRepository)) - self.assertEqual(repo.repo, repopath) - self.assertEqual(log_file_format(return_directory=True), logdir) - self.assertEqual(log_file_format(), logtmpl) - self.assertEqual(get_build_log_path(), tmplogdir) - - # redefine config file entries for proper testing below - buildpath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'build', 'path') - sourcepath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'source', 'path') - installpath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'install', 'path') - repopath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'repo', 'path') - logdir = 'somedir_custom' - logtmpl = 'test-custom-eb-%(name)_%(date)s%(time)s__%(version)s.log' - tmplogdir = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'tmplogdir') - softsuffix = 'myfavoritesoftware_custom' - modsuffix = 'modulesgohere_custom' - - configdict = { - 'buildpath': buildpath, - 'sourcepath': sourcepath, - 'installpath': installpath, - 'repopath': repopath, - 'logdir': logdir, - 'logtmpl': logtmpl, - 'tmplogdir': tmplogdir, - 'softsuffix': softsuffix, - 'modsuffix': modsuffix } - - # create custom config file, and point to it - mycustomconfigfile = os.path.join(self.tmpdir, 'mycustomconfig.py') - if not os.path.exists(os.path.dirname(mycustomconfigfile)): - os.makedirs(os.path.dirname(mycustomconfigfile)) - write_file(mycustomconfigfile, configtxt % configdict) - os.environ['EASYBUILDCONFIG'] = mycustomconfigfile - - # reconfigure - init_config() - cfg_fn = self.configure(args=[]) - - # verify configuration - self.assertEqual(cfg_fn, mycustomconfigfile) - self.assertEqual(build_path(), buildpath) - self.assertEqual(source_paths()[0], sourcepath) - self.assertEqual(install_path(), os.path.join(installpath, softsuffix)) - self.assertEqual(install_path(typ='mod'), os.path.join(installpath, modsuffix)) - repo = init_repository(get_repository(), get_repositorypath()) - self.assertTrue(isinstance(repo, FileRepository)) - self.assertEqual(repo.repo, repopath) - self.assertEqual(log_file_format(return_directory=True), logdir) - self.assertEqual(log_file_format(), logtmpl) - self.assertEqual(get_build_log_path(), tmplogdir) + self.assertEqual(config_options['tmpdir'], None) + self.assertEqual(config_options['tmp_logdir'], None) def test_generaloption_config(self): """Test new-style configuration (based on generaloption).""" @@ -366,6 +120,8 @@ def test_generaloption_config(self): options = init_config(args=[]) self.assertEqual(build_path(), buildpath_env_var) self.assertEqual(install_path(), os.path.join(prefix, 'software')) + self.assertEqual(get_repositorypath(), [os.path.join(prefix, 'ebfiles_repo')]) + del os.environ['EASYBUILD_PREFIX'] del os.environ['EASYBUILD_BUILDPATH'] @@ -378,7 +134,7 @@ def test_generaloption_config(self): write_file(config_file, '') args = [ - '--config', config_file, # force empty oldstyle config file + '--configfiles', config_file, # force empty config file '--prefix', prefix, '--installpath', install, '--repositorypath', repopath, @@ -391,14 +147,14 @@ def test_generaloption_config(self): self.assertEqual(install_path(typ='mod'), os.path.join(install, 'modules')) self.assertEqual(options.installpath, install) - self.assertEqual(options.config, config_file) + self.assertTrue(config_file in options.configfiles) # check mixed command line/env var configuration prefix = os.path.join(self.tmpdir, 'test3') install = os.path.join(self.tmpdir, 'test4', 'install') subdir_software = 'eb-soft' args = [ - '--config', config_file, # force empty oldstyle config file + '--configfiles', config_file, # force empty config file '--installpath', install, ] @@ -443,10 +199,22 @@ def test_generaloption_config_file(self): self.assertEqual(source_paths(), [os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'sources')]) # default self.assertEqual(install_path(), os.path.join(testpath2, 'software')) # via config file + # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory + # to check whether easyconfigs install path is auto-included in robot path + tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') + mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) + + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + shutil.copytree(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) + + orig_sys_path = sys.path[:] + sys.path.insert(0, tmpdir) # prepend to give it preference over possible other installed easyconfigs pkgs + # test with config file passed via environment variable cfgtxt = '\n'.join([ '[config]', 'buildpath = %s' % testpath1, + 'robot-paths = /tmp/foo:%(DEFAULT_ROBOT_PATHS)s', ]) write_file(config_file, cfgtxt) @@ -460,6 +228,8 @@ def test_generaloption_config_file(self): self.assertEqual(install_path(), os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'software')) # default self.assertEqual(source_paths(), [testpath2]) # via command line self.assertEqual(build_path(), testpath1) # via config file + self.assertTrue('/tmp/foo' in options.robot_paths) + self.assertTrue(os.path.join(tmpdir, 'easybuild', 'easyconfigs') in options.robot_paths) testpath3 = os.path.join(self.tmpdir, 'testTHREE') os.environ['EASYBUILD_SOURCEPATH'] = testpath2 @@ -474,6 +244,7 @@ def test_generaloption_config_file(self): self.assertEqual(build_path(), testpath1) # via config file del os.environ['EASYBUILD_CONFIGFILES'] + sys.path[:] = orig_sys_path def test_set_tmpdir(self): """Test set_tmpdir config function.""" @@ -495,6 +266,9 @@ def test_set_tmpdir(self): fd, tempfile_tmpfile = tempfile.mkstemp() self.assertTrue(tempfile_tmpfile.startswith(os.path.join(parent, 'easybuild-'))) + # tmp_logdir follows tmpdir + self.assertEqual(get_build_log_path(), mytmpdir) + # cleanup os.close(fd) shutil.rmtree(mytmpdir) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 6dcd861e96..2657389906 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -28,7 +28,6 @@ @author: Jens Timmerman (Ghent University) @author: Kenneth Hoste (Ghent University) """ -import copy import os import re import shutil @@ -44,7 +43,8 @@ from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools import config from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import mkdir, write_file +from easybuild.tools.filetools import mkdir, read_file, write_file +from easybuild.tools.modules import modules_tool class EasyBlockTest(EnhancedTestCase): @@ -77,23 +77,20 @@ def test_easyblock(self): def check_extra_options_format(extra_options): """Make sure extra_options value is of correct format.""" - # EasyBuild v1.x - self.assertTrue(isinstance(extra_options, list)) - for extra_option in extra_options: - self.assertTrue(isinstance(extra_option, tuple)) - self.assertEqual(len(extra_option), 2) - self.assertTrue(isinstance(extra_option[0], basestring)) - self.assertTrue(isinstance(extra_option[1], list)) - self.assertEqual(len(extra_option[1]), 3) - # EasyBuild v2.0 (breaks backward compatibility compared to v1.x) - #self.assertTrue(isinstance(extra_options, dict)) - #for key in extra_options: - # self.assertTrue(isinstance(extra_options[key], list)) - # self.assertTrue(len(extra_options[key]), 3) + # EasyBuild v2.0: dict with keys and values + # (breaks backward compatibility compared to v1.x) + self.assertTrue(isinstance(extra_options, dict)) # conversion to a dict works + extra_options.items() + extra_options.keys() + extra_options.values() + for key in extra_options.keys(): + self.assertTrue(isinstance(extra_options[key], list)) + self.assertTrue(len(extra_options[key]), 3) name = "pi" version = "3.14" - self.contents = '\n'.join([ + self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "%s"' % name, 'version = "%s"' % version, 'homepage = "http://example.com"', @@ -113,12 +110,18 @@ def check_extra_options_format(extra_options): sys.stdout.close() sys.stdout = stdoutorig + # check whether 'This is easyblock' log message is there + tup = ('EasyBlock', 'easybuild.framework.easyblock', '.*easybuild/framework/easyblock.pyc*') + eb_log_msg_re = re.compile(r"INFO This is easyblock %s from module %s (%s)" % tup, re.M) + logtxt = read_file(eb.logfile) + self.assertTrue(eb_log_msg_re.search(logtxt), "Pattern '%s' found in: %s" % (eb_log_msg_re.pattern, logtxt)) + # test extensioneasyblock, as extension exeb1 = ExtensionEasyBlock(eb, {'name': 'foo', 'version': '0.0'}) self.assertEqual(exeb1.cfg['name'], 'foo') extra_options = exeb1.extra_options() check_extra_options_format(extra_options) - self.assertTrue('options' in [key for (key, _) in extra_options]) + self.assertTrue('options' in extra_options) # test extensioneasyblock, as easyblock exeb2 = ExtensionEasyBlock(ec) @@ -126,18 +129,18 @@ def check_extra_options_format(extra_options): self.assertEqual(exeb2.cfg['version'], '3.14') extra_options = exeb2.extra_options() check_extra_options_format(extra_options) - self.assertTrue('options' in [key for (key, _) in extra_options]) + self.assertTrue('options' in extra_options) class TestExtension(ExtensionEasyBlock): @staticmethod def extra_options(): - return ExtensionEasyBlock.extra_options([('extra_param', [None, "help", CUSTOM])]) + return ExtensionEasyBlock.extra_options({'extra_param': [None, "help", CUSTOM]}) texeb = TestExtension(eb, {'name': 'bar'}) self.assertEqual(texeb.cfg['name'], 'bar') extra_options = texeb.extra_options() check_extra_options_format(extra_options) - self.assertTrue('options' in [key for (key, _) in extra_options]) - self.assertEqual([val for (key, val) in extra_options if key == 'extra_param'][0], [None, "help", CUSTOM]) + self.assertTrue('options' in extra_options) + self.assertEqual(extra_options['extra_param'], [None, "help", CUSTOM]) # cleanup eb.close_log() @@ -146,6 +149,7 @@ def extra_options(): def test_fake_module_load(self): """Testcase for fake module load""" self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', @@ -165,6 +169,7 @@ def test_fake_module_load(self): def test_make_module_req(self): """Testcase for make_module_req""" self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', @@ -200,6 +205,7 @@ def test_make_module_req(self): def test_extensions_step(self): """Test the extensions_step""" self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', @@ -216,7 +222,7 @@ def test_extensions_step(self): self.assertErrorRegex(EasyBuildError, "No default extension class set", eb.extensions_step, fetch=True) # test if everything works fine if set - self.contents += "\nexts_defaultclass = ['easybuild.framework.extension', 'Extension']" + self.contents += "\nexts_defaultclass = 'DummyExtension'" self.writeEC() eb = EasyBlock(EasyConfig(self.eb_file)) eb.builddir = config.build_path() @@ -234,19 +240,19 @@ def test_extensions_step(self): def test_skip_extensions_step(self): """Test the skip_extensions_step""" self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name": "dummy", "version": "dummy"}', 'exts_list = ["ext1", "ext2"]', - 'exts_filter = ("if [ %(name)s == \'ext2\' ]; then exit 0; else exit 1; fi", "")', - 'exts_defaultclass = ["easybuild.framework.extension", "Extension"]', + 'exts_filter = ("if [ %(ext_name)s == \'ext2\' ]; then exit 0; else exit 1; fi", "")', + 'exts_defaultclass = "DummyExtension"', ]) # check if skip skips correct extensions self.writeEC() eb = EasyBlock(EasyConfig(self.eb_file)) - #self.assertTrue('ext1' in eb.exts.keys() and 'ext2' in eb.exts.keys()) eb.builddir = config.build_path() eb.installdir = config.install_path() eb.skip = True @@ -266,15 +272,17 @@ def test_make_module_step(self): version = "3.14" deps = [('GCC', '4.6.4')] hiddendeps = [('toy', '0.0-deps')] + alldeps = deps + hiddendeps # hidden deps must be included in list of deps modextravars = {'PI': '3.1415', 'FOO': 'bar'} modextrapaths = {'PATH': 'pibin', 'CPATH': 'pi/include'} self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "%s"' % name, 'version = "%s"' % version, 'homepage = "http://example.com"', 'description = "test easyconfig"', "toolchain = {'name': 'dummy', 'version': 'dummy'}", - "dependencies = %s" % str(deps), + "dependencies = %s" % str(alldeps), "hiddendependencies = %s" % str(hiddendeps), "builddependencies = [('OpenMPI', '1.6.4-GCC-4.6.4')]", "modextravars = %s" % str(modextravars), @@ -288,7 +296,6 @@ def test_make_module_step(self): self.writeEC() ec = EasyConfig(self.eb_file) eb = EasyBlock(ec) - #eb.builddir = self.test_buildpath eb.installdir = os.path.join(config.install_path(), 'pi', '3.14') eb.check_readiness_step() @@ -320,6 +327,7 @@ def test_make_module_step(self): def test_gen_dirs(self): """Test methods that generate/set build/install directory names.""" self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', "name = 'pi'", "version = '3.14'", "homepage = 'http://example.com'", @@ -369,7 +377,7 @@ def test_get_easyblock_instance(self): testdir = os.path.abspath(os.path.dirname(__file__)) import easybuild eb_blocks_path = os.path.join(testdir, 'sandbox') - if not eb_blocks_path in sys.path: + if eb_blocks_path not in sys.path: sys.path.append(eb_blocks_path) easybuild = reload(easybuild) @@ -381,6 +389,51 @@ def test_get_easyblock_instance(self): eb = get_easyblock_instance(ec) self.assertTrue(isinstance(eb, EB_toy)) + # check whether 'This is easyblock' log message is there + tup = ('EB_toy', 'easybuild.easyblocks.toy', '.*test/framework/sandbox/easybuild/easyblocks/toy.pyc*') + eb_log_msg_re = re.compile(r"INFO This is easyblock %s from module %s (%s)" % tup, re.M) + logtxt = read_file(eb.logfile) + self.assertTrue(eb_log_msg_re.search(logtxt), "Pattern '%s' found in: %s" % (eb_log_msg_re.pattern, logtxt)) + + def test_fetch_patches(self): + """Test fetch_patches method.""" + # adjust PYTHONPATH such that test easyblocks are found + testdir = os.path.abspath(os.path.dirname(__file__)) + ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'toy-0.0.eb'))[0] + eb = get_easyblock_instance(ec) + + eb.fetch_patches() + self.assertEqual(len(eb.patches), 1) + self.assertEqual(eb.patches[0]['name'], 'toy-0.0_typo.patch') + self.assertFalse('level' in eb.patches[0]) + + # reset + eb.patches = [] + + patches = [ + ('toy-0.0_typo.patch', 0), # should also be level 0 (not None or something else) + ('toy-0.0_typo.patch', 4), # should be level 4 + ('toy-0.0_typo.patch', 'foobar'), # sourcepath should be set to 'foobar' + ('toy-0.0.tar.gz', 'some/path'), # copy mode (not a .patch file) + ] + # check if patch levels are parsed correctly + eb.fetch_patches(patch_specs=patches) + + self.assertEqual(len(eb.patches), 4) + self.assertEqual(eb.patches[0]['name'], 'toy-0.0_typo.patch') + self.assertEqual(eb.patches[0]['level'], 0) + self.assertEqual(eb.patches[1]['name'], 'toy-0.0_typo.patch') + self.assertEqual(eb.patches[1]['level'], 4) + self.assertEqual(eb.patches[2]['name'], 'toy-0.0_typo.patch') + self.assertEqual(eb.patches[2]['sourcepath'], 'foobar') + self.assertEqual(eb.patches[3]['name'], 'toy-0.0.tar.gz'), + self.assertEqual(eb.patches[3]['copy'], 'some/path') + + patches = [ + ('toy-0.0_level4.patch', False), # should throw an error, only int's an strings allowed here + ] + self.assertRaises(EasyBuildError, eb.fetch_patches, patch_specs=patches) + def test_obtain_file(self): """Test obtain_file method.""" toy_tarball = 'toy-0.0.tar.gz' @@ -398,7 +451,7 @@ def test_obtain_file(self): # 'downloading' a file to (first) sourcepath works init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, testdir)]) shutil.copy2(toy_tarball_path, tmpdir_subdir) - res = eb.obtain_file(toy_tarball, urls=[os.path.join('file://', tmpdir_subdir)]) + res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir]) self.assertEqual(res, os.path.join(tmpdir, 't', 'toy', toy_tarball)) # finding a file in sourcepath works @@ -407,16 +460,13 @@ def test_obtain_file(self): self.assertEqual(res, toy_tarball_path) # sourcepath has preference over downloading - res = eb.obtain_file(toy_tarball, urls=[os.path.join('file://', tmpdir_subdir)]) + res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir]) self.assertEqual(res, toy_tarball_path) # obtain_file yields error for non-existing files fn = 'thisisclearlyanonexistingfile' - try: - eb.obtain_file(fn, urls=[os.path.join('file://', tmpdir_subdir)]) - except EasyBuildError, err: - fail_regex = re.compile("Couldn't find file %s anywhere, and downloading it didn't work either" % fn) - self.assertTrue(fail_regex.search(str(err))) + error_regex = "Couldn't find file %s anywhere, and downloading it didn't work either" % fn + self.assertErrorRegex(EasyBuildError, error_regex, eb.obtain_file, fn, urls=['file://%s' % tmpdir_subdir]) # file specifications via URL also work, are downloaded to (first) sourcepath init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, sandbox_sources)]) @@ -465,6 +515,73 @@ def test_check_readiness(self): shutil.rmtree(tmpdir) + def test_exclude_path_to_top_of_module_tree(self): + """ + Make sure that modules under the HierarchicalMNS are correct, + w.r.t. not including any load statements for modules that build up the path to the top of the module tree. + """ + self.orig_module_naming_scheme = config.get_module_naming_scheme() + test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + all_stops = [x[0] for x in EasyBlock.get_steps()] + build_options = { + 'check_osdeps': False, + 'robot_path': [test_ecs_path], + 'valid_stops': all_stops, + 'validate': False, + } + os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS' + init_config(build_options=build_options) + self.setup_hierarchical_modules() + + modfile_prefix = os.path.join(self.test_installpath, 'modules', 'all') + mkdir(os.path.join(modfile_prefix, 'Compiler', 'GCC', '4.8.3'), parents=True) + mkdir(os.path.join(modfile_prefix, 'MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049'), parents=True) + + impi_modfile_path = os.path.join('Compiler', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049') + imkl_modfile_path = os.path.join('MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049', 'imkl', '11.1.2.144') + + # example: for imkl on top of iimpi toolchain with HierarchicalMNS, no module load statements should be included + # not for the toolchain or any of the toolchain components, + # since both icc/ifort and impi form the path to the top of the module tree + tests = [ + ('impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb', impi_modfile_path, ['icc', 'ifort', 'iccifort']), + ('imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb', imkl_modfile_path, ['icc', 'ifort', 'impi', 'iccifort', 'iimpi']), + ] + for ec_file, modfile_path, excluded_deps in tests: + ec = EasyConfig(os.path.join(test_ecs_path, ec_file)) + eb = EasyBlock(ec) + eb.toolchain.prepare() + modpath = eb.make_module_step() + modfile_path = os.path.join(modpath, modfile_path) + modtxt = read_file(modfile_path) + + for imkl_dep in excluded_deps: + tup = (imkl_dep, modfile_path, modtxt) + failmsg = "No 'module load' statement found for '%s' not found in module %s: %s" % tup + self.assertFalse(re.search("module load %s" % imkl_dep, modtxt), failmsg) + + os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme + init_config(build_options=build_options) + + def test_patch_step(self): + """Test patch step.""" + ec = process_easyconfig(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'toy-0.0.eb'))[0] + orig_sources = ec['ec']['sources'][:] + + # test applying patches without sources + ec['ec']['sources'] = [] + eb = EasyBlock(ec['ec']) + eb.fetch_step() + eb.extract_step() + self.assertErrorRegex(EasyBuildError, '.*', eb.patch_step) + + # test actual patching of unpacked sources + ec['ec']['sources'] = orig_sources + eb = EasyBlock(ec['ec']) + eb.fetch_step() + eb.extract_step() + eb.patch_step() + def tearDown(self): """ make sure to remove the temporary file """ super(EasyBlockTest, self).tearDown() diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index c78ec426a3..5f662ccbf2 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -29,11 +29,12 @@ @author: Kenneth Hoste (Ghent University) @author: Stijn De Weirdt (Ghent University) """ - +import copy import os import re import shutil import tempfile +from distutils.version import LooseVersion from test.framework.utilities import EnhancedTestCase, init_config from unittest import TestLoader, main from vsc.utils.fancylogger import setLogLevelDebug, logToScreen @@ -69,8 +70,6 @@ def setUp(self): if os.path.exists(self.eb_file): os.remove(self.eb_file) - self.orig_current_version = easybuild.tools.build_log.CURRENT_VERSION - def prep(self): """Prepare for test.""" # (re)cleanup last test file @@ -83,7 +82,6 @@ def prep(self): def tearDown(self): """ make sure to remove the temporary file """ - easybuild.tools.build_log.CURRENT_VERSION = self.orig_current_version super(EasyConfigTest, self).tearDown() if os.path.exists(self.eb_file): os.remove(self.eb_file) @@ -98,6 +96,7 @@ def test_empty(self): def test_mandatory(self): """ make sure all checking of mandatory variables works """ self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', ]) @@ -122,6 +121,7 @@ def test_mandatory(self): def test_validation(self): """ test other validations beside mandatory variables """ self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', @@ -151,15 +151,16 @@ def test_validation(self): self.prep() self.assertErrorRegex(EasyBuildError, "SyntaxError", EasyConfig, self.eb_file) - def test_shared_lib_ext(self): + def test_shlib_ext(self): """ inside easyconfigs shared_lib_ext should be set """ self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name":"dummy", "version": "dummy"}', - 'sanity_check_paths = { "files": ["lib/lib.%s" % shared_lib_ext] }', + 'sanity_check_paths = { "files": ["lib/lib.%s" % SHLIB_EXT] }', ]) self.prep() eb = EasyConfig(self.eb_file) @@ -168,6 +169,7 @@ def test_shared_lib_ext(self): def test_dependency(self): """ test all possible ways of specifying dependencies """ self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', @@ -214,6 +216,7 @@ def test_dependency(self): def test_extra_options(self): """ extra_options should allow other variables to be stored """ self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', @@ -224,7 +227,7 @@ def test_extra_options(self): ]) self.prep() eb = EasyConfig(self.eb_file) - self.assertRaises(KeyError, lambda: eb['custom_key']) + self.assertErrorRegex(EasyBuildError, "unknown easyconfig parameter", lambda: eb['custom_key']) extra_vars = {'custom_key': ['default', "This is a default key", easyconfig.CUSTOM]} @@ -247,13 +250,8 @@ def test_extra_options(self): # test if extra toolchain options are being passed self.assertEqual(eb.toolchain.options['static'], True) - # test legacy behavior of passing a list of tuples rather than a dict - eb = EasyConfig(self.eb_file, extra_options=extra_vars.items()) - self.assertEqual(eb['custom_key'], 'test') - - extra_vars.update({'mandatory_key': ['default', 'another mandatory key', easyconfig.MANDATORY]}) - # test extra mandatory vars + extra_vars.update({'mandatory_key': ['default', 'another mandatory key', easyconfig.MANDATORY]}) self.assertErrorRegex(EasyBuildError, r"mandatory variables? \S* not provided", EasyConfig, self.eb_file, extra_vars) @@ -269,6 +267,7 @@ def test_exts_list(self): os.environ['EASYBUILD_SOURCEPATH'] = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') init_config() self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', @@ -284,8 +283,8 @@ def test_exts_list(self): ' "source_urls": [("http://example.com", "suffix")],' ' "patches": ["toy-0.0.eb"],', # dummy patch to avoid downloading fail ' "checksums": [', - ' "504c7036558938f997c1c269a01d7458",', # checksum for source (gzip-1.4.eb) - ' "ddd5161154f5db67701525123129ff09",', # checksum for patch (toy-0.0.eb) + ' "787393bfc465c85607a5b24486e861c5",', # MD5 checksum for source (gzip-1.4.eb) + ' "ddd5161154f5db67701525123129ff09",', # MD5 checksum for patch (toy-0.0.eb) ' ],', ' }),', ']', @@ -298,6 +297,7 @@ def test_exts_list(self): def test_suggestions(self): """ If a typo is present, suggestions should be provided (if possible) """ self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', @@ -321,6 +321,7 @@ def test_tweaking(self): os.close(fd) patches = ["t1.patch", ("t2.patch", 1), ("t3.patch", "test"), ("t4.h", "include")] self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'homepage = "http://www.example.com"', 'description = "dummy description"', @@ -416,24 +417,6 @@ def test_installversion(self): installver = det_full_ec_version(cfg) self.assertEqual(installver, correct_installver) - def test_legacy_installversion(self): - """Test generation of install version (legacy).""" - - ver = "3.14" - verpref = "myprefix|" - versuff = "|mysuffix" - tcname = "GCC" - tcver = "4.6.3" - dummy = "dummy" - - correct_installver = "%s%s-%s-%s%s" % (verpref, ver, tcname, tcver, versuff) - installver = det_installversion(ver, tcname, tcver, verpref, versuff) - self.assertEqual(installver, correct_installver) - - correct_installver = "%s%s%s" % (verpref, ver, versuff) - installver = det_installversion(ver, dummy, tcver, verpref, versuff) - self.assertEqual(installver, correct_installver) - def test_obtain_easyconfig(self): """test obtaining an easyconfig file given certain specifications""" @@ -447,41 +430,51 @@ def test_obtain_easyconfig(self): "pi-3.15-GCC-4.3.2.eb", "pi-3.15-GCC-4.4.5.eb", "foo-1.2.3-GCC-4.3.2.eb"] - eb_files = [(fns[0], "\n".join(['name = "pi"', - 'version = "3.12"', - 'homepage = "http://example.com"', - 'description = "test easyconfig"', - 'toolchain = {"name": "dummy", "version": "dummy"}', - 'patches = %s' % patches - ])), - (fns[1], "\n".join(['name = "pi"', - 'version = "3.13"', - 'homepage = "http://example.com"', - 'description = "test easyconfig"', - 'toolchain = {"name": "%s", "version": "%s"}' % (tcname, tcver), - 'patches = %s' % patches - ])), - (fns[2], "\n".join(['name = "pi"', - 'version = "3.15"', - 'homepage = "http://example.com"', - 'description = "test easyconfig"', - 'toolchain = {"name": "%s", "version": "%s"}' % (tcname, tcver), - 'patches = %s' % patches - ])), - (fns[3], "\n".join(['name = "pi"', - 'version = "3.15"', - 'homepage = "http://example.com"', - 'description = "test easyconfig"', - 'toolchain = {"name": "%s", "version": "4.5.1"}' % tcname, - 'patches = %s' % patches - ])), - (fns[4], "\n".join(['name = "foo"', - 'version = "1.2.3"', - 'homepage = "http://example.com"', - 'description = "test easyconfig"', - 'toolchain = {"name": "%s", "version": "%s"}' % (tcname, tcver), - 'foo_extra1 = "bar"', - ])) + eb_files = [(fns[0], "\n".join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.12"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = {"name": "dummy", "version": "dummy"}', + 'patches = %s' % patches + ])), + (fns[1], "\n".join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.13"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = {"name": "%s", "version": "%s"}' % (tcname, tcver), + 'patches = %s' % patches + ])), + (fns[2], "\n".join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.15"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = {"name": "%s", "version": "%s"}' % (tcname, tcver), + 'patches = %s' % patches + ])), + (fns[3], "\n".join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.15"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = {"name": "%s", "version": "4.5.1"}' % tcname, + 'patches = %s' % patches + ])), + (fns[4], "\n".join([ + 'easyblock = "ConfigureMake"', + 'name = "foo"', + 'version = "1.2.3"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = {"name": "%s", "version": "%s"}' % (tcname, tcver), + 'foo_extra1 = "bar"', + ])) ] @@ -496,7 +489,10 @@ def test_obtain_easyconfig(self): self.assertErrorRegex(EasyBuildError, error_regexp, obtain_ec_for, specs, [self.ec_dir], None) # should find matching easyconfig file - specs = {'name': 'foo', 'version': '1.2.3'} + specs = { + 'name': 'foo', + 'version': '1.2.3' + } res = obtain_ec_for(specs, [self.ec_dir], None) self.assertEqual(res[0], False) self.assertEqual(res[1], os.path.join(self.ec_dir, fns[-1])) @@ -543,7 +539,7 @@ def test_obtain_easyconfig(self): ec = EasyConfig(res[1]) self.assertEqual(ec['version'], specs['version']) txt = read_file(res[1]) - self.assertTrue(re.search("version = [\"']%s[\"'] .*was: [\"']3.13[\"']" % ver, txt)) + self.assertTrue(re.search("^version = [\"']%s[\"']$" % ver, txt, re.M)) os.remove(res[1]) # should pick correct toolchain version as well, i.e. now newer than what's specified, if a choice needs to be made @@ -557,8 +553,8 @@ def test_obtain_easyconfig(self): self.assertEqual(ec['version'], specs['version']) self.assertEqual(ec['toolchain']['version'], specs['toolchain_version']) txt = read_file(res[1]) - pattern = "toolchain = .*version.*[\"']%s[\"'].*was: .*version.*[\"']%s[\"']" % (specs['toolchain_version'], tcver) - self.assertTrue(re.search(pattern, txt)) + pattern = "^toolchain = .*version.*[\"']%s[\"'].*}$" % specs['toolchain_version'] + self.assertTrue(re.search(pattern, txt, re.M)) os.remove(res[1]) @@ -601,6 +597,15 @@ def test_obtain_easyconfig(self): 'hidden': True, }, ] + + # hidden dependencies must be included in list of dependencies + res = obtain_ec_for(specs, [self.ec_dir], None) + self.assertEqual(res[0], True) + error_pattern = "Hidden dependencies with visible module names .* not in list of dependencies: .*" + self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, res[1]) + + specs['dependencies'].append(('test', '3.2.1')) + res = obtain_ec_for(specs, [self.ec_dir], None) self.assertEqual(res[0], True) ec = EasyConfig(res[1]) @@ -671,6 +676,7 @@ def test_templating(self): } # don't use any escaping insanity here, since it is templated itself self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "%(name)s"', 'version = "%(version)s"', 'homepage = "http://example.com/%%(nameletter)s/%%(nameletterlower)s"', @@ -723,6 +729,7 @@ def test_constant_doc(self): def test_build_options(self): """Test configure/build/install options, both strings and lists.""" orig_contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', @@ -792,6 +799,7 @@ def test_build_options(self): def test_buildininstalldir(self): """Test specifying build in install dir.""" self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', @@ -870,8 +878,10 @@ def test_get_easyblock_class(self): ]: self.assertEqual(get_easyblock_class(easyblock), easyblock_class) - self.assertEqual(get_easyblock_class(None, name='gzip'), ConfigureMake) + self.assertEqual(get_easyblock_class(None, name='gzip', default_fallback=False), None) self.assertEqual(get_easyblock_class(None, name='toy'), EB_toy) + self.assertErrorRegex(EasyBuildError, "Failed to import EB_TOY", get_easyblock_class, None, name='TOY') + self.assertEqual(get_easyblock_class(None, name='TOY', error_on_failed_import=False), None) def test_easyconfig_paths(self): """Test create_paths function.""" @@ -884,27 +894,6 @@ def test_easyconfig_paths(self): ] self.assertEqual(cand_paths, expected_paths) - def test_deprecated_options(self): - """Test whether deprecated options are handled correctly.""" - deprecated_options = [ - ('makeopts', 'buildopts', 'CC=foo'), - ('premakeopts', 'prebuildopts', ['PATH=%(builddir)s/foo:$PATH', 'PATH=%(builddir)s/bar:$PATH']), - ] - clean_contents = [ - 'name = "pi"', - 'version = "3.14"', - 'homepage = "http://example.com"', - 'description = "test easyconfig"', - 'toolchain = {"name": "dummy", "version": "dummy"}', - 'buildininstalldir = True', - ] - # alternative option is ready to use - for depr_opt, new_opt, val in deprecated_options: - self.contents = '\n'.join(clean_contents + ['%s = %s' % (depr_opt, quote_str(val))]) - self.prep() - ec = EasyConfig(self.eb_file) - self.assertEqual(ec[depr_opt], ec[new_opt]) - def test_toolchain_inspection(self): """Test whether available toolchain inspection functionality is working.""" build_options = { @@ -952,6 +941,86 @@ def test_filter_deps(self): opts = init_config(args=['--filter-deps=zlib,ncurses']) self.assertEqual(opts.filter_deps, ['zlib', 'ncurses']) + def test_replaced_easyconfig_parameters(self): + """Test handling of replaced easyconfig parameters.""" + test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs') + ec = EasyConfig(os.path.join(test_ecs_dir, 'toy-0.0.eb')) + replaced_parameters = { + 'license': ('software_license', '2.0'), + 'makeopts': ('buildopts', '2.0'), + 'premakeopts': ('prebuildopts', '2.0'), + } + for key, (newkey, ver) in replaced_parameters.items(): + error_regex = "NO LONGER SUPPORTED since v%s.*'%s' is replaced by '%s'" % (ver, key, newkey) + self.assertErrorRegex(EasyBuildError, error_regex, ec.get, key) + self.assertErrorRegex(EasyBuildError, error_regex, lambda k: ec[k], key) + def foo(key): + ec[key] = 'foo' + self.assertErrorRegex(EasyBuildError, error_regex, foo, key) + + def test_deprecated_easyconfig_parameters(self): + """Test handling of replaced easyconfig parameters.""" + os.environ.pop('EASYBUILD_DEPRECATED') + easybuild.tools.build_log.CURRENT_VERSION = self.orig_current_version + init_config() + + test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs') + ec = EasyConfig(os.path.join(test_ecs_dir, 'toy-0.0.eb')) + + orig_deprecated_parameters = copy.deepcopy(easyconfig.easyconfig.DEPRECATED_PARAMETERS) + easyconfig.easyconfig.DEPRECATED_PARAMETERS.update({ + 'foobar': ('barfoo', '0.0'), # deprecated since forever + 'foobarbarfoo': ('barfoofoobar', '1000000000'), # won't be actually deprecated for a while + }) + + # copy classes before reloading, so we can restore them (other isinstance checks fail) + orig_EasyConfig = copy.deepcopy(easyconfig.easyconfig.EasyConfig) + orig_ActiveMNS = copy.deepcopy(easyconfig.easyconfig.ActiveMNS) + reload(easyconfig.easyconfig) + + for key, (newkey, depr_ver) in easyconfig.easyconfig.DEPRECATED_PARAMETERS.items(): + if LooseVersion(depr_ver) <= easybuild.tools.build_log.CURRENT_VERSION: + # deprecation error + error_regex = "DEPRECATED.*since v%s.*'%s' is deprecated.*use '%s' instead" % (depr_ver, key, newkey) + self.assertErrorRegex(EasyBuildError, error_regex, ec.get, key) + self.assertErrorRegex(EasyBuildError, error_regex, lambda k: ec[k], key) + def foo(key): + ec[key] = 'foo' + self.assertErrorRegex(EasyBuildError, error_regex, foo, key) + else: + # only deprecation warning, but key is replaced when getting/setting + ec[key] = 'test123' + self.assertEqual(ec[newkey], 'test123') + self.assertEqual(ec[key], 'test123') + ec[newkey] = '123test' + self.assertEqual(ec[newkey], '123test') + self.assertEqual(ec[key], '123test') + + easyconfig.easyconfig.DEPRECATED_PARAMETERS = orig_deprecated_parameters + reload(easyconfig.easyconfig) + easyconfig.easyconfig.EasyConfig = orig_EasyConfig + easyconfig.easyconfig.ActiveMNS = orig_ActiveMNS + + def test_unknown_easyconfig_parameter(self): + """Check behaviour when unknown easyconfig parameters are used.""" + self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.14"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = {"name": "dummy", "version": "dummy"}', + ]) + self.prep() + ec = EasyConfig(self.eb_file) + self.assertFalse('therenosucheasyconfigparameterlikethis' in ec) + error_regex = "unknown easyconfig parameter" + self.assertErrorRegex(EasyBuildError, error_regex, lambda k: ec[k], 'therenosucheasyconfigparameterlikethis') + def set_ec_key(key): + """Dummy function to set easyconfig parameter in 'ec' EasyConfig instance""" + ec[key] = 'foobar' + self.assertErrorRegex(EasyBuildError, error_regex, set_ec_key, 'therenosucheasyconfigparameterlikethis') + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/easyconfigs/CUDA-5.5.22-GCC-4.8.2.eb b/test/framework/easyconfigs/CUDA-5.5.22-GCC-4.8.2.eb new file mode 100644 index 0000000000..00bf4df2f9 --- /dev/null +++ b/test/framework/easyconfigs/CUDA-5.5.22-GCC-4.8.2.eb @@ -0,0 +1,31 @@ +## +# This file is an EasyBuild reciPY as per https://github.com/hpcugent/easybuild +# +# Copyright:: Copyright 2012-2014 Cyprus Institute / CaSToRC, Uni.Lu/LCSB, NTUA, Ghent University +# Authors:: George Tsouloupas , Fotis Georgatos , Kenneth Hoste +# License:: MIT/GPL +# $Id$ +# +# This work implements a part of the HPCBIOS project and is a component of the policy: +# http://hpcbios.readthedocs.org/en/latest/HPCBIOS_2012-99.html +## +# should be EB_CUDA, but OK for testing purposes +easyblock = 'EB_toy' + +name = 'CUDA' +version = '5.5.22' + +homepage = 'https://developer.nvidia.com/cuda-toolkit' +description = """CUDA (formerly Compute Unified Device Architecture) is a parallel + computing platform and programming model created by NVIDIA and implemented by the + graphics processing units (GPUs) that they produce. CUDA gives developers access + to the virtual instruction set and memory of the parallel computational elements in CUDA GPUs.""" + +toolchain = {'name': 'GCC', 'version': '4.8.2'} + +# eg. http://developer.download.nvidia.com/compute/cuda/5_5/rel/installers/cuda_5.5.22_linux_64.run +source_urls = ['http://developer.download.nvidia.com/compute/cuda/5_5/rel/installers/'] + +sources = ['%(namelower)s_%(version)s_linux_64.run'] + +moduleclass = 'system' diff --git a/test/framework/easyconfigs/FFTW-3.3.3-gompi-1.4.10.eb b/test/framework/easyconfigs/FFTW-3.3.3-gompi-1.4.10.eb index 06b0c2e2e1..bead8318f4 100644 --- a/test/framework/easyconfigs/FFTW-3.3.3-gompi-1.4.10.eb +++ b/test/framework/easyconfigs/FFTW-3.3.3-gompi-1.4.10.eb @@ -1,3 +1,5 @@ +easyblock = 'ConfigureMake' + name = 'FFTW' version = '3.3.3' diff --git a/test/framework/easyconfigs/GCC-4.6.3.eb b/test/framework/easyconfigs/GCC-4.6.3.eb index 3b4c4c53c9..8f9e3c6a1f 100644 --- a/test/framework/easyconfigs/GCC-4.6.3.eb +++ b/test/framework/easyconfigs/GCC-4.6.3.eb @@ -1,3 +1,6 @@ +# should be EB_GCC, but OK for testing purposes +easyblock = 'EB_toy' + name="GCC" version='4.6.3' diff --git a/test/framework/easyconfigs/GCC-4.6.4.eb b/test/framework/easyconfigs/GCC-4.6.4.eb index bf4adc61a6..baf448818b 100644 --- a/test/framework/easyconfigs/GCC-4.6.4.eb +++ b/test/framework/easyconfigs/GCC-4.6.4.eb @@ -1,3 +1,6 @@ +# should be EB_GCC, but OK for testing purposes +easyblock = 'EB_toy' + name = "GCC" version = '4.6.4' diff --git a/test/framework/easyconfigs/GCC-4.7.2.eb b/test/framework/easyconfigs/GCC-4.7.2.eb index 7b4dfcf410..d4b386baae 100644 --- a/test/framework/easyconfigs/GCC-4.7.2.eb +++ b/test/framework/easyconfigs/GCC-4.7.2.eb @@ -1,3 +1,6 @@ +# should be EB_GCC, but OK for testing purposes +easyblock = 'EB_toy' + name = "GCC" version = '4.7.2' diff --git a/test/framework/easyconfigs/GCC-4.8.2.eb b/test/framework/easyconfigs/GCC-4.8.2.eb new file mode 100644 index 0000000000..a7723b5eb9 --- /dev/null +++ b/test/framework/easyconfigs/GCC-4.8.2.eb @@ -0,0 +1,31 @@ +# should be EB_GCC, but OK for testing purposes +easyblock = 'EB_toy' + +name = "GCC" +version = '4.8.2' + +homepage = 'http://gcc.gnu.org/' +description = """The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...).""" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +source_urls = [ + 'http://ftpmirror.gnu.org/%(namelower)s/%(namelower)s-%(version)s', # GCC auto-resolving HTTP mirror + 'http://ftpmirror.gnu.org/gmp', # idem for GMP + 'http://ftpmirror.gnu.org/mpfr', # idem for MPFR + 'http://www.multiprecision.org/mpc/download', # MPC official +] +sources = [ + SOURCELOWER_TAR_GZ, + 'gmp-5.1.3.tar.bz2', + 'mpfr-3.1.2.tar.gz', + 'mpc-1.0.1.tar.gz', +] + +languages = ['c', 'c++', 'fortran', 'lto'] + +# building GCC sometimes fails if make parallelism is too high, so let's limit it +maxparallel = 4 + +moduleclass = 'compiler' diff --git a/test/framework/easyconfigs/GCC-4.8.3.eb b/test/framework/easyconfigs/GCC-4.8.3.eb new file mode 100644 index 0000000000..14e91d37d2 --- /dev/null +++ b/test/framework/easyconfigs/GCC-4.8.3.eb @@ -0,0 +1,31 @@ +# should be EB_GCC, but OK for testing purposes +easyblock = 'EB_toy' + +name = "GCC" +version = '4.8.3' + +homepage = 'http://gcc.gnu.org/' +description = """The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...).""" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +source_urls = [ + 'http://ftpmirror.gnu.org/%(namelower)s/%(namelower)s-%(version)s', # GCC auto-resolving HTTP mirror + 'http://ftpmirror.gnu.org/gmp', # idem for GMP + 'http://ftpmirror.gnu.org/mpfr', # idem for MPFR + 'http://www.multiprecision.org/mpc/download', # MPC official +] +sources = [ + SOURCELOWER_TAR_GZ, + 'gmp-5.1.3.tar.bz2', + 'mpfr-3.1.2.tar.gz', + 'mpc-1.0.1.tar.gz', +] + +languages = ['c', 'c++', 'fortran', 'lto'] + +# building GCC sometimes fails if make parallelism is too high, so let's limit it +maxparallel = 4 + +moduleclass = 'compiler' diff --git a/test/framework/easyconfigs/GCC-4.9.2.eb b/test/framework/easyconfigs/GCC-4.9.2.eb new file mode 100644 index 0000000000..ec651b931d --- /dev/null +++ b/test/framework/easyconfigs/GCC-4.9.2.eb @@ -0,0 +1,36 @@ +# should be EB_GCC, but OK for testing purposes +easyblock = 'EB_toy' + +name = "GCC" +version = '4.9.2' + +homepage = 'http://gcc.gnu.org/' +description = """The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...).""" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +source_urls = [ + 'http://ftpmirror.gnu.org/%(namelower)s/%(namelower)s-%(version)s', # GCC auto-resolving HTTP mirror + 'http://ftpmirror.gnu.org/gmp', # idem for GMP + 'http://ftpmirror.gnu.org/mpfr', # idem for MPFR + 'http://www.multiprecision.org/mpc/download', # MPC official +] + +mpfr_version = '3.1.2' + +sources = [ + SOURCELOWER_TAR_BZ2, + 'gmp-6.0.0a.tar.bz2', + 'mpfr-%s.tar.gz' % mpfr_version, + 'mpc-1.0.2.tar.gz', +] + +patches = [('mpfr-%s-allpatches-20140630.patch' % mpfr_version, '../mpfr-%s' % mpfr_version)] + +languages = ['c', 'c++', 'fortran', 'lto'] + +# building GCC sometimes fails if make parallelism is too high, so let's limit it +maxparallel = 4 + +moduleclass = 'compiler' diff --git a/test/framework/easyconfigs/OpenBLAS-0.2.6-gompi-1.4.10-LAPACK-3.4.2.eb b/test/framework/easyconfigs/OpenBLAS-0.2.6-gompi-1.4.10-LAPACK-3.4.2.eb index 8f1b31ff70..bd9785f7cc 100644 --- a/test/framework/easyconfigs/OpenBLAS-0.2.6-gompi-1.4.10-LAPACK-3.4.2.eb +++ b/test/framework/easyconfigs/OpenBLAS-0.2.6-gompi-1.4.10-LAPACK-3.4.2.eb @@ -1,3 +1,5 @@ +easyblock = 'ConfigureMake' + name = 'OpenBLAS' version = '0.2.6' @@ -44,7 +46,7 @@ installopts = threading + " PREFIX=%(installdir)s" sanity_check_paths = { 'files': ['include/cblas.h', 'include/f77blas.h', 'include/lapacke_config.h', 'include/lapacke.h', 'include/lapacke_mangling.h', 'include/lapacke_utils.h', 'include/openblas_config.h', - 'lib/libopenblas.a', 'lib/libopenblas.%s' % shared_lib_ext], + 'lib/libopenblas.a', 'lib/libopenblas.%s' % SHLIB_EXT], 'dirs': [], } diff --git a/test/framework/easyconfigs/OpenMPI-1.6.4-GCC-4.6.4.eb b/test/framework/easyconfigs/OpenMPI-1.6.4-GCC-4.6.4.eb index 053791e834..bd0832e690 100644 --- a/test/framework/easyconfigs/OpenMPI-1.6.4-GCC-4.6.4.eb +++ b/test/framework/easyconfigs/OpenMPI-1.6.4-GCC-4.6.4.eb @@ -1,3 +1,5 @@ +easyblock = 'ConfigureMake' + name = 'OpenMPI' version = '1.6.4' @@ -25,7 +27,7 @@ else: sanity_check_paths = { 'files': ["bin/%s" % binfile for binfile in ["ompi_info", "opal_wrapper", "orterun"]] + - ["lib/lib%s.%s" % (libfile, shared_lib_ext) for libfile in ["mpi_cxx", "mpi_f77", "mpi_f90", + ["lib/lib%s.%s" % (libfile, SHLIB_EXT) for libfile in ["mpi_cxx", "mpi_f77", "mpi_f90", "mpi", "ompitrace", "open-pal", "open-rte", "vt", "vt-hyb", "vt-mpi", "vt-mpi-unify"]] + diff --git a/test/framework/easyconfigs/OpenMPI-1.6.4-GCC-4.7.2.eb b/test/framework/easyconfigs/OpenMPI-1.6.4-GCC-4.7.2.eb index fa3425f1ab..1505eba3ad 100644 --- a/test/framework/easyconfigs/OpenMPI-1.6.4-GCC-4.7.2.eb +++ b/test/framework/easyconfigs/OpenMPI-1.6.4-GCC-4.7.2.eb @@ -1,3 +1,5 @@ +easyblock = 'ConfigureMake' + name = 'OpenMPI' version = '1.6.4' @@ -25,7 +27,7 @@ else: sanity_check_paths = { 'files': ["bin/%s" % binfile for binfile in ["ompi_info", "opal_wrapper", "orterun"]] + - ["lib/lib%s.%s" % (libfile, shared_lib_ext) for libfile in ["mpi_cxx", "mpi_f77", "mpi_f90", + ["lib/lib%s.%s" % (libfile, SHLIB_EXT) for libfile in ["mpi_cxx", "mpi_f77", "mpi_f90", "mpi", "ompitrace", "open-pal", "open-rte", "vt", "vt-hyb", "vt-mpi", "vt-mpi-unify"]] + diff --git a/test/framework/easyconfigs/ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb b/test/framework/easyconfigs/ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb index 8f7ad295d1..14f049732b 100644 --- a/test/framework/easyconfigs/ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb +++ b/test/framework/easyconfigs/ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb @@ -1,3 +1,6 @@ +# should be EB_ScaLAPACK, but OK for testing purposes +easyblock = 'EB_toy' + name = 'ScaLAPACK' version = '2.0.2' diff --git a/test/framework/easyconfigs/gzip-1.4-GCC-4.6.3.eb b/test/framework/easyconfigs/gzip-1.4-GCC-4.6.3.eb index b59e6160d2..c5f783e816 100644 --- a/test/framework/easyconfigs/gzip-1.4-GCC-4.6.3.eb +++ b/test/framework/easyconfigs/gzip-1.4-GCC-4.6.3.eb @@ -9,6 +9,7 @@ # This work implements a part of the HPCBIOS project and is a component of the policy: # http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html ## +easyblock = 'ConfigureMake' name = 'gzip' version = '1.4' @@ -26,6 +27,7 @@ sources = ['%s-%s.tar.gz'%(name,version)] source_urls = [GNU_SOURCE] hiddendependencies = [('toy', '0.0', '-deps', True)] +dependencies = hiddendependencies # hidden deps must be included in list of deps # make sure the gzip and gunzip binaries are available after installation sanity_check_paths = { diff --git a/test/framework/easyconfigs/gzip-1.4.eb b/test/framework/easyconfigs/gzip-1.4.eb index ab769c6b69..c5a94274b3 100644 --- a/test/framework/easyconfigs/gzip-1.4.eb +++ b/test/framework/easyconfigs/gzip-1.4.eb @@ -9,6 +9,7 @@ # This work implements a part of the HPCBIOS project and is a component of the policy: # http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html ## +easyblock = 'ConfigureMake' name = 'gzip' version = '1.4' diff --git a/test/framework/easyconfigs/gzip-1.5-goolf-1.4.10.eb b/test/framework/easyconfigs/gzip-1.5-goolf-1.4.10.eb index 08ce4ddc61..d1636586c9 100644 --- a/test/framework/easyconfigs/gzip-1.5-goolf-1.4.10.eb +++ b/test/framework/easyconfigs/gzip-1.5-goolf-1.4.10.eb @@ -9,6 +9,7 @@ # This work implements a part of the HPCBIOS project and is a component of the policy: # http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html ## +easyblock = 'ConfigureMake' name = 'gzip' version = '1.5' diff --git a/test/framework/easyconfigs/gzip-1.5-ictce-4.1.13.eb b/test/framework/easyconfigs/gzip-1.5-ictce-4.1.13.eb index 2552d296e9..9fb11ce6ca 100644 --- a/test/framework/easyconfigs/gzip-1.5-ictce-4.1.13.eb +++ b/test/framework/easyconfigs/gzip-1.5-ictce-4.1.13.eb @@ -9,6 +9,7 @@ # This work implements a part of the HPCBIOS project and is a component of the policy: # http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html ## +easyblock = 'ConfigureMake' name = 'gzip' version = '1.5' diff --git a/test/framework/easyconfigs/hwloc-1.6.2-GCC-4.6.4.eb b/test/framework/easyconfigs/hwloc-1.6.2-GCC-4.6.4.eb index 116ee4cb8c..3c07bbe614 100644 --- a/test/framework/easyconfigs/hwloc-1.6.2-GCC-4.6.4.eb +++ b/test/framework/easyconfigs/hwloc-1.6.2-GCC-4.6.4.eb @@ -1,3 +1,5 @@ +easyblock = 'ConfigureMake' + name = 'hwloc' version = '1.6.2' diff --git a/test/framework/easyconfigs/hwloc-1.6.2-GCC-4.7.2.eb b/test/framework/easyconfigs/hwloc-1.6.2-GCC-4.7.2.eb index 00a3ce7444..19d34c3d1a 100644 --- a/test/framework/easyconfigs/hwloc-1.6.2-GCC-4.7.2.eb +++ b/test/framework/easyconfigs/hwloc-1.6.2-GCC-4.7.2.eb @@ -1,3 +1,5 @@ +easyblock = 'ConfigureMake' + name = 'hwloc' version = '1.6.2' diff --git a/test/framework/easyconfigs/icc-2013.5.192-GCC-4.8.3.eb b/test/framework/easyconfigs/icc-2013.5.192-GCC-4.8.3.eb new file mode 100644 index 0000000000..a084c374d0 --- /dev/null +++ b/test/framework/easyconfigs/icc-2013.5.192-GCC-4.8.3.eb @@ -0,0 +1,26 @@ +# should be EB_icc, but OK for testing purposes +easyblock = 'EB_toy' + +name = 'icc' +version = '2013.5.192' + +homepage = 'http://software.intel.com/en-us/intel-compilers/' +description = "C and C++ compiler from Intel" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +sources = ['l_ccompxe_%(version)s.tgz'] + +gcc = 'GCC' +gccver = '4.8.3' +versionsuffix = '-%s-%s' % (gcc, gccver) + +dependencies = [(gcc, gccver)] + +dontcreateinstalldir = 'True' + +# license file +import os +license_file = os.path.join(os.getenv('HOME'), "licenses", "intel", "license.lic") + +moduleclass = 'compiler' diff --git a/test/framework/easyconfigs/iccifort-2013.5.192-GCC-4.8.3.eb b/test/framework/easyconfigs/iccifort-2013.5.192-GCC-4.8.3.eb new file mode 100644 index 0000000000..9a152f81fe --- /dev/null +++ b/test/framework/easyconfigs/iccifort-2013.5.192-GCC-4.8.3.eb @@ -0,0 +1,17 @@ +easyblock = "Toolchain" + +name = 'iccifort' +version = '2013.5.192' +versionsuffix = '-GCC-4.8.3' + +homepage = 'http://software.intel.com/en-us/intel-cluster-toolkit-compiler/' +description = """Intel C, C++ and Fortran compilers""" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +dependencies = [ + ('icc', version, versionsuffix), + ('ifort', version, versionsuffix), +] + +moduleclass = 'toolchain' diff --git a/test/framework/easyconfigs/ifort-2013.3.163.eb b/test/framework/easyconfigs/ifort-2013.3.163.eb new file mode 100644 index 0000000000..09c286af0b --- /dev/null +++ b/test/framework/easyconfigs/ifort-2013.3.163.eb @@ -0,0 +1,20 @@ +# should be EB_ifort, but OK for testing purposes +easyblock = 'EB_toy' + +name = 'ifort' +version = '2013.3.163' + +homepage = 'http://software.intel.com/en-us/intel-compilers/' +description = "Fortran compiler from Intel" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +sources = ['l_fcompxe_%s.tgz' % version] + +dontcreateinstalldir = 'True' + +# license file +import os +license_file = os.path.join(os.getenv('HOME'), "licenses", "intel", "license.lic") + +moduleclass = 'compiler' diff --git a/test/framework/easyconfigs/ifort-2013.5.192-GCC-4.8.3.eb b/test/framework/easyconfigs/ifort-2013.5.192-GCC-4.8.3.eb new file mode 100644 index 0000000000..8a74093865 --- /dev/null +++ b/test/framework/easyconfigs/ifort-2013.5.192-GCC-4.8.3.eb @@ -0,0 +1,26 @@ +# should be EB_ifort, but OK for testing purposes +easyblock = 'EB_toy' + +name = 'ifort' +version = '2013.5.192' + +homepage = 'http://software.intel.com/en-us/intel-compilers/' +description = "Fortran compiler from Intel" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +sources = ['l_fcompxe_%(version)s.tgz'] + +gcc = 'GCC' +gccver = '4.8.3' +versionsuffix = '-%s-%s' % (gcc, gccver) + +dependencies = [(gcc, gccver)] + +dontcreateinstalldir = 'True' + +# license file +import os +license_file = os.path.join(os.getenv('HOME'), "licenses", "intel", "license.lic") + +moduleclass = 'compiler' diff --git a/test/framework/easyconfigs/iimpi-5.5.3-GCC-4.8.3.eb b/test/framework/easyconfigs/iimpi-5.5.3-GCC-4.8.3.eb new file mode 100644 index 0000000000..221f1fb36d --- /dev/null +++ b/test/framework/easyconfigs/iimpi-5.5.3-GCC-4.8.3.eb @@ -0,0 +1,21 @@ +easyblock = "Toolchain" + +name = 'iimpi' +version = '5.5.3' +versionsuffix = '-GCC-4.8.3' + +homepage = 'http://software.intel.com/en-us/intel-cluster-toolkit-compiler/' +description = """Intel C/C++ and Fortran compilers, alongside Intel MPI.""" + +toolchain = {'name': 'dummy', 'version': 'dummy'} + +suff = '5.192' +compver = '2013.%s' % suff + +dependencies = [ # version/released + ('icc', compver, versionsuffix), # 28 Apr 2014 + ('ifort', compver, versionsuffix), # 28 Apr 2014 + ('impi', '4.1.3.049', '', ('iccifort', '%s%s' % (compver, versionsuffix))), # 06 Mar 2014 +] + +moduleclass = 'toolchain' diff --git a/test/framework/easyconfigs/imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb b/test/framework/easyconfigs/imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb new file mode 100644 index 0000000000..a23cf2c3c9 --- /dev/null +++ b/test/framework/easyconfigs/imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb @@ -0,0 +1,25 @@ +# should be EB_imkl, but OK for testing purposes +easyblock = 'EB_toy' + +name = 'imkl' +version = '11.1.2.144' + +homepage = 'http://software.intel.com/en-us/intel-mkl/' +description = """Intel Math Kernel Library is a library of highly optimized, + extensively threaded math routines for science, engineering, and financial + applications that require maximum performance. Core math functions include + BLAS, LAPACK, ScaLAPACK, Sparse Solvers, Fast Fourier Transforms, Vector Math, and more.""" + +toolchain = {'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'} + +sources = ['l_mkl_%(version)s.tgz'] + +dontcreateinstalldir = 'True' + +interfaces = True + +# license file +import os +license_file = os.path.join(os.getenv('HOME'), "licenses", "intel", "license.lic") + +moduleclass = 'numlib' diff --git a/test/framework/easyconfigs/impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb b/test/framework/easyconfigs/impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb new file mode 100644 index 0000000000..e325ca2e75 --- /dev/null +++ b/test/framework/easyconfigs/impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb @@ -0,0 +1,22 @@ +# should be EB_impi, but OK for testing purposes +easyblock = 'EB_toy' + +name = 'impi' +version = '4.1.3.049' + +homepage = 'http://software.intel.com/en-us/intel-mpi-library/' +description = """The Intel(R) MPI Library for Linux* OS is a multi-fabric message + passing library based on ANL MPICH2 and OSU MVAPICH2. The Intel MPI Library for + Linux OS implements the Message Passing Interface, version 2 (MPI-2) specification.""" + +toolchain = {'name': 'iccifort', 'version': '2013.5.192-GCC-4.8.3'} + +sources = ['l_mpi_p_%(version)s.tgz'] + +dontcreateinstalldir = 'True' + +# license file +import os +license_file = os.path.join(os.getenv('HOME'), "licenses", "intel", "license.lic") + +moduleclass = 'mpi' diff --git a/test/framework/easyconfigs/impi-4.1.3.049.eb b/test/framework/easyconfigs/impi-4.1.3.049.eb index e55725a62a..4267dce6b6 100644 --- a/test/framework/easyconfigs/impi-4.1.3.049.eb +++ b/test/framework/easyconfigs/impi-4.1.3.049.eb @@ -1,3 +1,6 @@ +# should be EB_impi, but OK for testing purposes +easyblock = 'EB_toy' + name = 'impi' version = '4.1.3.049' diff --git a/test/framework/easyconfigs/v1.0/GCC-4.6.3.eb b/test/framework/easyconfigs/v1.0/GCC-4.6.3.eb index 3b4c4c53c9..8f9e3c6a1f 100644 --- a/test/framework/easyconfigs/v1.0/GCC-4.6.3.eb +++ b/test/framework/easyconfigs/v1.0/GCC-4.6.3.eb @@ -1,3 +1,6 @@ +# should be EB_GCC, but OK for testing purposes +easyblock = 'EB_toy' + name="GCC" version='4.6.3' diff --git a/test/framework/easyconfigs/v1.0/gzip-1.4-GCC-4.6.3.eb b/test/framework/easyconfigs/v1.0/gzip-1.4-GCC-4.6.3.eb index 9f1c615c51..0f8d2efc28 100644 --- a/test/framework/easyconfigs/v1.0/gzip-1.4-GCC-4.6.3.eb +++ b/test/framework/easyconfigs/v1.0/gzip-1.4-GCC-4.6.3.eb @@ -9,6 +9,7 @@ # This work implements a part of the HPCBIOS project and is a component of the policy: # http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html ## +easyblock = 'ConfigureMake' name = 'gzip' version = '1.4' diff --git a/test/framework/easyconfigs/v1.0/gzip-1.4.eb b/test/framework/easyconfigs/v1.0/gzip-1.4.eb index ab769c6b69..c5a94274b3 100644 --- a/test/framework/easyconfigs/v1.0/gzip-1.4.eb +++ b/test/framework/easyconfigs/v1.0/gzip-1.4.eb @@ -9,6 +9,7 @@ # This work implements a part of the HPCBIOS project and is a component of the policy: # http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html ## +easyblock = 'ConfigureMake' name = 'gzip' version = '1.4' diff --git a/test/framework/easyconfigs/v1.0/gzip-1.5-goolf-1.4.10.eb b/test/framework/easyconfigs/v1.0/gzip-1.5-goolf-1.4.10.eb index 08ce4ddc61..d1636586c9 100644 --- a/test/framework/easyconfigs/v1.0/gzip-1.5-goolf-1.4.10.eb +++ b/test/framework/easyconfigs/v1.0/gzip-1.5-goolf-1.4.10.eb @@ -9,6 +9,7 @@ # This work implements a part of the HPCBIOS project and is a component of the policy: # http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html ## +easyblock = 'ConfigureMake' name = 'gzip' version = '1.5' diff --git a/test/framework/easyconfigs/v1.0/gzip-1.5-ictce-4.1.13.eb b/test/framework/easyconfigs/v1.0/gzip-1.5-ictce-4.1.13.eb index 2552d296e9..9fb11ce6ca 100644 --- a/test/framework/easyconfigs/v1.0/gzip-1.5-ictce-4.1.13.eb +++ b/test/framework/easyconfigs/v1.0/gzip-1.5-ictce-4.1.13.eb @@ -9,6 +9,7 @@ # This work implements a part of the HPCBIOS project and is a component of the policy: # http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html ## +easyblock = 'ConfigureMake' name = 'gzip' version = '1.5' diff --git a/test/framework/easyconfigs/v2.0/GCC.eb b/test/framework/easyconfigs/v2.0/GCC.eb index 6a1baf1bef..b753004a6e 100644 --- a/test/framework/easyconfigs/v2.0/GCC.eb +++ b/test/framework/easyconfigs/v2.0/GCC.eb @@ -5,6 +5,9 @@ docstring test @author: Stijn De Weirdt (UGent) @maintainer: Kenneth Hoste (UGent) """ +# should be EB_GCC, but OK for testing purposes +easyblock = 'EB_toy' + name = "GCC" homepage = 'http://gcc.gnu.org/' diff --git a/test/framework/easyconfigs/v2.0/doesnotexist.eb b/test/framework/easyconfigs/v2.0/doesnotexist.eb index 7067a83a07..7a85355ce6 100644 --- a/test/framework/easyconfigs/v2.0/doesnotexist.eb +++ b/test/framework/easyconfigs/v2.0/doesnotexist.eb @@ -5,6 +5,7 @@ docstring test @author: Stijn De Weirdt (UGent) @maintainer: Kenneth Hoste (UGent) """ +easyblock = 'ConfigureMake' name = 'doesnotexist' diff --git a/test/framework/easyconfigs/v2.0/gzip.eb b/test/framework/easyconfigs/v2.0/gzip.eb index de795268c7..602528e2af 100644 --- a/test/framework/easyconfigs/v2.0/gzip.eb +++ b/test/framework/easyconfigs/v2.0/gzip.eb @@ -41,4 +41,5 @@ versions = 1.4, 1.5 toolchains = dummy == dummy, goolf, GCC == 4.6.3, goolf == 1.4.10, ictce == 4.1.13 [DEFAULT] +easyblock = ConfigureMake moduleclass = base diff --git a/test/framework/easyconfigs/v2.0/libpng.eb b/test/framework/easyconfigs/v2.0/libpng.eb index 1cc9175bca..ecc5bd59f6 100644 --- a/test/framework/easyconfigs/v2.0/libpng.eb +++ b/test/framework/easyconfigs/v2.0/libpng.eb @@ -28,6 +28,7 @@ versions = 1.5.10, 1.5.11, 1.5.13, 1.5.14, 1.6.2, 1.6.3, 1.6.6 toolchains = goolf == 1.4.10, ictce == 4.1.13, goalf == 1.1.0-no-OFED [DEFAULT] +easyblock = ConfigureMake moduleclass = lib [DEPENDENCIES] diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 651ea77c30..6c94e81a5d 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -33,7 +33,8 @@ import shutil import stat import tempfile -from test.framework.utilities import EnhancedTestCase, find_full_path +import urllib2 +from test.framework.utilities import EnhancedTestCase from unittest import TestLoader, main import easybuild.tools.filetools as ft @@ -51,17 +52,6 @@ class FileToolsTest(EnhancedTestCase): ('0_foo+0x0x#-$__', 'EB_0_underscore_foo_plus_0x0x_hash__minus__dollar__underscore__underscore_'), ] - def setUp(self): - """Set up testcase.""" - super(FileToolsTest, self).setUp() - self.legacySetUp() - - def legacySetUp(self): - self.log.deprecated("legacySetUp", "2.0") - cfg_path = os.path.join('easybuild', 'easybuild_config.py') - cfg_full_path = find_full_path(cfg_path) - self.assertTrue(cfg_full_path) - def test_extract_cmd(self): """Test various extract commands.""" tests = [ @@ -190,9 +180,26 @@ def test_download_file(self): target_location = os.path.join(self.test_buildpath, 'some', 'subdir', fn) # provide local file path as source URL test_dir = os.path.abspath(os.path.dirname(__file__)) - source_url = os.path.join('file://', test_dir, 'sandbox', 'sources', 'toy', fn) + source_url = 'file://%s/sandbox/sources/toy/%s' % (test_dir, fn) + res = ft.download_file(fn, source_url, target_location) + self.assertEqual(res, target_location, "'download' of local file works") + + # non-existing files result in None return value + self.assertEqual(ft.download_file(fn, 'file://%s/nosuchfile' % test_dir, target_location), None) + + # install broken proxy handler for opening local files + # this should make urllib2.urlopen use this broken proxy for downloading from a file:// URL + proxy_handler = urllib2.ProxyHandler({'file': 'file://%s/nosuchfile' % test_dir}) + urllib2.install_opener(urllib2.build_opener(proxy_handler)) + + # downloading over a broken proxy results in None return value (failed download) + # this tests whether proxies are taken into account by download_file + self.assertEqual(ft.download_file(fn, source_url, target_location), None, "download over broken proxy fails") + + # restore a working file handler, and retest download of local file + urllib2.install_opener(urllib2.build_opener(urllib2.FileHandler())) res = ft.download_file(fn, source_url, target_location) - self.assertEqual(res, target_location) + self.assertEqual(res, target_location, "'download' of local file works after removing broken proxy") def test_mkdir(self): """Test mkdir function.""" diff --git a/test/framework/format_convert.py b/test/framework/format_convert.py index 4cb5e70f3b..84c5c0f067 100644 --- a/test/framework/format_convert.py +++ b/test/framework/format_convert.py @@ -61,6 +61,12 @@ def test_listofstrings(self): res = ListOfStrings(txt.replace(ListOfStrings.SEPARATOR_LIST, ListOfStrings.SEPARATOR_LIST + ' ')) self.assertEqual(res, dest) + # empty string yields a list with an empty string + self.assertEqual(ListOfStrings(''), ['']) + + # empty entries are retained + self.assertEqual(ListOfStrings('a,,b'), ['a', '', 'b']) + def test_dictofstrings(self): """Test dict of strings""" # test default separators diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index a65e8f5781..3bd0b376a4 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -171,8 +171,15 @@ def test_alias(self): def test_load_msg(self): """Test including a load message in the module file.""" - tcl_load_msg = '\nif [ module-info mode load ] {\n puts stderr "test"\n}\n' - self.assertEqual(tcl_load_msg, self.modgen.msg_on_load('test')) + tcl_load_msg = '\n'.join([ + '', + "if [ module-info mode load ] {", + " puts stderr \"test \\$test \\$test", + "test \\$foo \\$bar\"", + "}", + '', + ]) + self.assertEqual(tcl_load_msg, self.modgen.msg_on_load('test $test \\$test\ntest $foo \\$bar')) def test_tcl_footer(self): """Test including a Tcl footer.""" @@ -252,7 +259,7 @@ def test_mns(): init_config(build_options=build_options) err_pattern = 'nosucheasyconfigparameteravailable' - self.assertErrorRegex(KeyError, err_pattern, EasyConfig, os.path.join(ecs_dir, 'gzip-1.5-goolf-1.4.10.eb')) + self.assertErrorRegex(EasyBuildError, err_pattern, EasyConfig, os.path.join(ecs_dir, 'gzip-1.5-goolf-1.4.10.eb')) # test simple custom module naming scheme os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'TestModuleNamingScheme' @@ -393,11 +400,19 @@ def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, init_modpaths): init_config(build_options=build_options) # format: easyconfig_file: (short_mod_name, mod_subdir, modpath_extensions) + iccver = '2013.5.192-GCC-4.8.3' + impi_ec = 'impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb' + imkl_ec = 'imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb' test_ecs = { 'GCC-4.7.2.eb': ('GCC/4.7.2', 'Core', ['Compiler/GCC/4.7.2'], ['Core']), 'OpenMPI-1.6.4-GCC-4.7.2.eb': ('OpenMPI/1.6.4', 'Compiler/GCC/4.7.2', ['MPI/GCC/4.7.2/OpenMPI/1.6.4'], ['Core']), 'gzip-1.5-goolf-1.4.10.eb': ('gzip/1.5', 'MPI/GCC/4.7.2/OpenMPI/1.6.4', [], ['Core']), 'goolf-1.4.10.eb': ('goolf/1.4.10', 'Core', [], ['Core']), + 'icc-2013.5.192-GCC-4.8.3.eb': ('icc/%s' % iccver, 'Core', ['Compiler/intel/%s' % iccver], ['Core']), + 'ifort-2013.3.163.eb': ('ifort/2013.3.163', 'Core', ['Compiler/intel/2013.3.163'], ['Core']), + 'CUDA-5.5.22-GCC-4.8.2.eb': ('CUDA/5.5.22', 'Compiler/GCC/4.8.2', ['Compiler/GCC-CUDA/4.8.2-5.5.22'], ['Core']), + impi_ec: ('impi/4.1.3.049', 'Compiler/intel/%s' % iccver, ['MPI/intel/%s/impi/4.1.3.049' % iccver], ['Core']), + imkl_ec: ('imkl/11.1.2.144', 'MPI/intel/%s/impi/4.1.3.049' % iccver, [], ['Core']), } for ecfile, mns_vals in test_ecs.items(): test_ec(ecfile, *mns_vals) @@ -419,6 +434,7 @@ def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, init_modpaths): for ecfile, mns_vals in test_ecs.items(): test_ec(ecfile, *mns_vals) + def suite(): """ returns all the testcases in this module """ return TestLoader().loadTestsFromTestCase(ModuleGeneratorTest) diff --git a/test/framework/modules.py b/test/framework/modules.py index ae8ab63f6f..00cce786d7 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -43,7 +43,7 @@ # number of modules included for testing purposes -TEST_MODULES_COUNT = 44 +TEST_MODULES_COUNT = 50 class ModulesTest(EnhancedTestCase): @@ -122,12 +122,6 @@ def test_exists(self): 'Compiler/GCC/4.7.2/OpenMPI/1.6.4', 'toy/.0.0-deps'] self.assertEqual(self.testmods.exist(mod_names), [True, False, False, False, True, True, True]) - # test deprecated functionality - self.assertTrue(self.testmods.exists('OpenMPI/1.6.4-GCC-4.6.4')) - self.assertFalse(self.testmods.exists('foo/1.2.3')) - # exists should not return True for incomplete module names - self.assertFalse(self.testmods.exists('GCC')) - def test_load(self): """ test if we load one module it is in the loaded_modules """ self.init_testmods() @@ -283,6 +277,7 @@ def test_path_to_top_of_module_tree(self): path = modtool.path_to_top_of_module_tree(init_modpaths, 'FFTW/3.3.3', full_mod_subdir, deps) self.assertEqual(path, ['OpenMPI/1.6.4', 'GCC/4.7.2']) + def suite(): """ returns all the testcases in this module """ return TestLoader().loadTestsFromTestCase(ModulesTest) diff --git a/test/framework/modules/Compiler/intel/2013.5.192-GCC-4.8.3/impi/4.1.3.049 b/test/framework/modules/Compiler/intel/2013.5.192-GCC-4.8.3/impi/4.1.3.049 new file mode 100644 index 0000000000..6b65daae4f --- /dev/null +++ b/test/framework/modules/Compiler/intel/2013.5.192-GCC-4.8.3/impi/4.1.3.049 @@ -0,0 +1,30 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { The Intel(R) MPI Library for Linux* OS is a multi-fabric message + passing library based on ANL MPICH2 and OSU MVAPICH2. The Intel MPI Library for + Linux OS implements the Message Passing Interface, version 2 (MPI-2) specification. - Homepage: http://software.intel.com/en-us/intel-mpi-library/ + } +} + +module-whatis {Description: The Intel(R) MPI Library for Linux* OS is a multi-fabric message + passing library based on ANL MPICH2 and OSU MVAPICH2. The Intel MPI Library for + Linux OS implements the Message Passing Interface, version 2 (MPI-2) specification. - Homepage: http://software.intel.com/en-us/intel-mpi-library/} + +set root /tmp/software/Compiler/intel/2013.5.192/impi/4.1.3.049 + +conflict impi +module use /tmp/modules/all/MPI/intel/2013.5.192/impi/4.1.3.049 +prepend-path CPATH $root/include64 +prepend-path LD_LIBRARY_PATH $root/lib64 +prepend-path LIBRARY_PATH $root/lib64 +prepend-path PATH $root/bin64 + +setenv EBROOTIMPI "$root" +setenv EBVERSIONIMPI "4.1.3.049" +setenv EBDEVELIMPI "$root/easybuild/Compiler-intel-2013.5.192-impi-4.1.3.049-easybuild-devel" + +prepend-path INTEL_LICENSE_FILE /tmp/license.lic +setenv I_MPI_ROOT $root + +# Built with EasyBuild version 1.16.0dev diff --git a/test/framework/modules/Core/GCC/4.8.3 b/test/framework/modules/Core/GCC/4.8.3 new file mode 100644 index 0000000000..4ddfb7dc18 --- /dev/null +++ b/test/framework/modules/Core/GCC/4.8.3 @@ -0,0 +1,30 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...). - Homepage: http://gcc.gnu.org/ + } +} + +module-whatis {Description: The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...). - Homepage: http://gcc.gnu.org/} + +set root /tmp/software/Core/GCC/4.8.3 + +conflict GCC +module use /tmp/modules/all/Compiler/GCC/4.8.3 +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LD_LIBRARY_PATH $root/lib64 +prepend-path LD_LIBRARY_PATH $root/lib/gcc/x86_64-unknown-linux-gnu/4.8.3 +prepend-path LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib64 +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin + +setenv EBROOTGCC "$root" +setenv EBVERSIONGCC "4.8.3" +setenv EBDEVELGCC "$root/easybuild/Core-GCC-4.8.3-easybuild-devel" + + +# Built with EasyBuild version 1.16.0dev diff --git a/test/framework/modules/Core/icc/2013.5.192-GCC-4.8.3 b/test/framework/modules/Core/icc/2013.5.192-GCC-4.8.3 new file mode 100644 index 0000000000..4428aa9709 --- /dev/null +++ b/test/framework/modules/Core/icc/2013.5.192-GCC-4.8.3 @@ -0,0 +1,34 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { C and C++ compiler from Intel - Homepage: http://software.intel.com/en-us/intel-compilers/ + } +} + +module-whatis {Description: C and C++ compiler from Intel - Homepage: http://software.intel.com/en-us/intel-compilers/} + +set root /tmp/software/Core/icc/2013.5.192-GCC-4.8.3 + +conflict icc +module use /tmp/modules/all/Compiler/intel/2013.5.192-GCC-4.8.3 + +if { ![is-loaded GCC/4.8.3] } { + module load GCC/4.8.3 +} + +prepend-path IDB_HOME $root/bin/intel64 +prepend-path LD_LIBRARY_PATH $root/compiler/lib +prepend-path LD_LIBRARY_PATH $root/compiler/lib/intel64 +prepend-path MANPATH $root/man +prepend-path MANPATH $root/man/en_US +prepend-path PATH $root/bin +prepend-path PATH $root/bin/intel64 + +setenv EBROOTICC "$root" +setenv EBVERSIONICC "2013.5.192" +setenv EBDEVELICC "$root/easybuild/Core-icc-2013.5.192-GCC-4.8.3-easybuild-devel" + +prepend-path INTEL_LICENSE_FILE /tmp/license.lic +prepend-path NLSPATH $root/idb/intel64/locale/%l_%t/%N + +# Built with EasyBuild version 1.16.0dev diff --git a/test/framework/modules/Core/iccifort/2013.5.192-GCC-4.8.3 b/test/framework/modules/Core/iccifort/2013.5.192-GCC-4.8.3 new file mode 100644 index 0000000000..2375404a69 --- /dev/null +++ b/test/framework/modules/Core/iccifort/2013.5.192-GCC-4.8.3 @@ -0,0 +1,28 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { Intel C, C++ and Fortran compilers - Homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/ + } +} + +module-whatis {Description: Intel C, C++ and Fortran compilers - Homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/} + +set root /tmp/software/Core/iccifort/2013.5.192-GCC-4.8.3 + +conflict iccifort + +if { ![is-loaded icc/2013.5.192-GCC-4.8.3] } { + module load icc/2013.5.192-GCC-4.8.3 +} + +if { ![is-loaded ifort/2013.5.192-GCC-4.8.3] } { + module load ifort/2013.5.192-GCC-4.8.3 +} + + +setenv EBROOTICCIFORT "$root" +setenv EBVERSIONICCIFORT "2013.5.192" +setenv EBDEVELICCIFORT "$root/easybuild/Core-iccifort-2013.5.192-GCC-4.8.3-easybuild-devel" + + +# Built with EasyBuild version 1.16.0dev diff --git a/test/framework/modules/Core/ifort/2013.5.192-GCC-4.8.3 b/test/framework/modules/Core/ifort/2013.5.192-GCC-4.8.3 new file mode 100644 index 0000000000..750f0a4df9 --- /dev/null +++ b/test/framework/modules/Core/ifort/2013.5.192-GCC-4.8.3 @@ -0,0 +1,34 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { Fortran compiler from Intel - Homepage: http://software.intel.com/en-us/intel-compilers/ + } +} + +module-whatis {Description: Fortran compiler from Intel - Homepage: http://software.intel.com/en-us/intel-compilers/} + +set root /tmp/software/Core/ifort/2013.5.192-GCC-4.8.3 + +conflict ifort +module use /tmp/modules/all/Compiler/intel/2013.5.192-GCC-4.8.3 + +if { ![is-loaded GCC/4.8.3] } { + module load GCC/4.8.3 +} + +prepend-path IDB_HOME $root/bin/intel64 +prepend-path LD_LIBRARY_PATH $root/compiler/lib +prepend-path LD_LIBRARY_PATH $root/compiler/lib/intel64 +prepend-path MANPATH $root/man +prepend-path MANPATH $root/man/en_US +prepend-path PATH $root/bin +prepend-path PATH $root/bin/intel64 + +setenv EBROOTIFORT "$root" +setenv EBVERSIONIFORT "2013.5.192" +setenv EBDEVELIFORT "$root/easybuild/Core-ifort-2013.5.192-GCC-4.8.3-easybuild-devel" + +prepend-path INTEL_LICENSE_FILE /tmp/license.lic +prepend-path NLSPATH $root/idb/intel64/locale/%l_%t/%N + +# Built with EasyBuild version 1.16.0dev diff --git a/test/framework/modules/Core/iimpi/5.5.3-GCC-4.8.3 b/test/framework/modules/Core/iimpi/5.5.3-GCC-4.8.3 new file mode 100644 index 0000000000..3bb0b860cb --- /dev/null +++ b/test/framework/modules/Core/iimpi/5.5.3-GCC-4.8.3 @@ -0,0 +1,32 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { Intel C/C++ and Fortran compilers, alongside Intel MPI. - Homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/ + } +} + +module-whatis {Description: Intel C/C++ and Fortran compilers, alongside Intel MPI. - Homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/} + +set root /tmp/software/Core/iimpi/5.5.3-GCC-4.8.3 + +conflict iimpi + +if { ![is-loaded icc/2013.5.192-GCC-4.8.3] } { + module load icc/2013.5.192-GCC-4.8.3 +} + +if { ![is-loaded ifort/2013.5.192-GCC-4.8.3] } { + module load ifort/2013.5.192-GCC-4.8.3 +} + +if { ![is-loaded impi/4.1.3.049] } { + module load impi/4.1.3.049 +} + + +setenv EBROOTIIMPI "$root" +setenv EBVERSIONIIMPI "5.5.3" +setenv EBDEVELIIMPI "$root/easybuild/Core-iimpi-5.5.3-GCC-4.8.3-easybuild-devel" + + +# Built with EasyBuild version 1.16.0dev diff --git a/test/framework/modules/cgompi/1.1.6 b/test/framework/modules/cgompi/1.1.6 index 22bd608d51..33367e0a50 100644 --- a/test/framework/modules/cgompi/1.1.6 +++ b/test/framework/modules/cgompi/1.1.6 @@ -13,8 +13,12 @@ set root /user/scratch/gent/vsc400/vsc40023/easybuild_REGTEST/SL6/sandybridge conflict cgompi -if { ![is-loaded ClangGCC/1.1.2] } { - module load ClangGCC/1.1.2 +if { ![is-loaded GCC/4.7.2] } { + module load GCC/4.7.2 +} + +if { ![is-loaded Clang/3.2-GCC-4.7.2] } { + module load Clang/3.2-GCC-4.7.2 } if { ![is-loaded OpenMPI/1.6.4-ClangGCC-1.1.2] } { diff --git a/test/framework/modules/cgoolf/1.1.6 b/test/framework/modules/cgoolf/1.1.6 index 477da80d19..7ce5aa2cc0 100644 --- a/test/framework/modules/cgoolf/1.1.6 +++ b/test/framework/modules/cgoolf/1.1.6 @@ -13,8 +13,12 @@ set root /user/scratch/gent/vsc400/vsc40023/easybuild_REGTEST/SL6/sandybridge conflict cgoolf -if { ![is-loaded ClangGCC/1.1.2] } { - module load ClangGCC/1.1.2 +if { ![is-loaded GCC/4.7.2] } { + module load GCC/4.7.2 +} + +if { ![is-loaded Clang/3.2-GCC-4.7.2] } { + module load Clang/3.2-GCC-4.7.2 } if { ![is-loaded OpenMPI/1.6.4-ClangGCC-1.1.2] } { diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index ad8c28de27..5b92f8822a 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -76,7 +76,7 @@ def test_mock(self): os.environ['module'] = "() { eval `/bin/echo $*`\n}" # ue empty mod_path list, otherwise the install_path is called - mmt = MockModulesTool(mod_paths=[]) + mmt = MockModulesTool(mod_paths=[], testing=True) # the version of the MMT is the commandline option self.assertEqual(mmt.version, StrictVersion(MockModulesTool.VERSION_OPTION)) @@ -91,7 +91,7 @@ def test_environment_command(self): os.environ['module'] = "() { %s $*\n}" % BrokenMockModulesTool.COMMAND try: - bmmt = BrokenMockModulesTool(mod_paths=[]) + bmmt = BrokenMockModulesTool(mod_paths=[], testing=True) # should never get here self.assertTrue(False, 'BrokenMockModulesTool should fail') except EasyBuildError, err: @@ -100,7 +100,7 @@ def test_environment_command(self): os.environ[BrokenMockModulesTool.COMMAND_ENVIRONMENT] = MockModulesTool.COMMAND os.environ['module'] = "() { /bin/echo $*\n}" BrokenMockModulesTool._instances.pop(BrokenMockModulesTool, None) - bmmt = BrokenMockModulesTool(mod_paths=[]) + bmmt = BrokenMockModulesTool(mod_paths=[], testing=True) cmd_abspath = which(MockModulesTool.COMMAND) self.assertEqual(bmmt.version, StrictVersion(MockModulesTool.VERSION_OPTION)) @@ -112,8 +112,9 @@ def test_environment_command(self): def test_module_mismatch(self): """Test whether mismatch detection between modules tool and 'module' function works.""" # redefine 'module' function (deliberate mismatch with used module command in MockModulesTool) - os.environ['module'] = "() { eval `/Users/kehoste/Modules/$MODULE_VERSION/bin/modulecmd bash $*`\n}" - self.assertErrorRegex(EasyBuildError, ".*pattern .* not found in defined 'module' function", MockModulesTool) + os.environ['module'] = "() { eval `/tmp/Modules/$MODULE_VERSION/bin/modulecmd bash $*`\n}" + error_regex = ".*pattern .* not found in defined 'module' function" + self.assertErrorRegex(EasyBuildError, error_regex, MockModulesTool, testing=True) # check whether escaping error by allowing mismatch via build options works build_options = { @@ -123,7 +124,7 @@ def test_module_mismatch(self): fancylogger.logToFile(self.logfile) - mt = MockModulesTool() + mt = MockModulesTool(testing=True) f = open(self.logfile, 'r') logtxt = f.read() f.close() @@ -133,13 +134,13 @@ def test_module_mismatch(self): # redefine 'module' function with correct module command os.environ['module'] = "() { eval `/bin/echo $*`\n}" MockModulesTool._instances.pop(MockModulesTool) - mt = MockModulesTool() + mt = MockModulesTool(testing=True) self.assertTrue(isinstance(mt.loaded_modules(), list)) # dummy usage # a warning should be logged if the 'module' function is undefined del os.environ['module'] MockModulesTool._instances.pop(MockModulesTool) - mt = MockModulesTool() + mt = MockModulesTool(testing=True) f = open(self.logfile, 'r') logtxt = f.read() f.close() @@ -173,8 +174,7 @@ def test_lmod_specific(self): # initialize Lmod modules tool, pass full path to 'lmod' via $LMOD_CMD os.environ['LMOD_CMD'] = lmod_abspath - lmod = Lmod() - lmod.testing = True + lmod = Lmod(testing=True) # obtain list of availabe modules, should be non-empty self.assertTrue(lmod.available(), "List of available modules obtained using Lmod is non-empty") diff --git a/test/framework/options.py b/test/framework/options.py index 6fab6cf364..66f75a88a1 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -33,16 +33,17 @@ import shutil import sys import tempfile -from test.framework.utilities import EnhancedTestCase +from test.framework.utilities import EnhancedTestCase, init_config from unittest import TestLoader from unittest import main as unittestmain +from urllib2 import URLError import easybuild.tools.build_log from easybuild.framework.easyconfig import BUILD, CUSTOM, DEPENDENCIES, EXTENSIONS, FILEMANAGEMENT, LICENSE from easybuild.framework.easyconfig import MANDATORY, MODULES, OTHER, TOOLCHAIN from easybuild.tools.build_log import EasyBuildError from easybuild.tools.environment import modify_env -from easybuild.tools.filetools import read_file, write_file +from easybuild.tools.filetools import mkdir, read_file, write_file from easybuild.tools.modules import modules_tool from easybuild.tools.options import EasyBuildOptions from easybuild.tools.version import VERSION @@ -103,12 +104,11 @@ def test_no_args(self): def test_debug(self): """Test enabling debug logging.""" - for debug_arg in ['-d', '--debug']: args = [ - '--software-name=somethingrandom', - debug_arg, - ] + 'nosuchfile.eb', + debug_arg, + ] outtxt = self.eb_main(args) for log_msg_type in ['DEBUG', 'INFO', 'ERROR']: @@ -123,7 +123,7 @@ def test_info(self): for info_arg in ['--info']: args = [ - '--software-name=somethingrandom', + 'nosuchfile.eb', info_arg, ] outtxt = self.eb_main(args) @@ -144,7 +144,7 @@ def test_quiet(self): for quiet_arg in ['--quiet']: args = [ - '--software-name=somethingrandom', + 'nosuchfile.eb', quiet_arg, ] outtxt = self.eb_main(args) @@ -261,11 +261,10 @@ def test_job(self): # use gzip-1.4.eb easyconfig file that comes with the tests eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.4.eb') - # check log message with --job - for job_args in [ # options passed are reordered, so order here matters to make tests pass - ['--debug'], - ['--debug', '--stop=configure', '--try-software-name=foo'], - ]: + def check_args(job_args, passed_args=None): + """Check whether specified args yield expected result.""" + if passed_args is None: + passed_args = job_args[:] # clear log file write_file(self.logfile, '') @@ -276,13 +275,20 @@ def test_job(self): ] + job_args outtxt = self.eb_main(args) - job_msg = "INFO.* Command template for jobs: .* && eb %%\(spec\)s.* %s.*\n" % ' .*'.join(job_args) - assertmsg = "Info log message with job command template when using --job (job_msg: %s, outtxt: %s)" % (job_msg, outtxt) + job_msg = "INFO.* Command template for jobs: .* && eb %%\(spec\)s.* %s.*\n" % ' .*'.join(passed_args) + assertmsg = "Info log msg with job command template for --job (job_msg: %s, outtxt: %s)" % (job_msg, outtxt) self.assertTrue(re.search(job_msg, outtxt), assertmsg) modify_env(os.environ, self.orig_environ) tempfile.tempdir = None + # options passed are reordered, so order here matters to make tests pass + check_args(['--debug']) + check_args(['--debug', '--stop=configure', '--try-software-name=foo']) + check_args(['--debug', '--robot-paths=/tmp/foo:/tmp/bar']) + # --robot has preference over --robot-paths, --robot is not passed down + check_args(['--debug', '--robot-paths=/tmp/foo', '--robot=/tmp/bar'], passed_args=['--debug', '--robot-paths=/tmp/bar:/tmp/foo']) + # 'zzz' prefix in the test name is intentional to make this test run last, # since it fiddles with the logging infrastructure which may break things def test_zzz_logtostdout(self): @@ -328,24 +334,27 @@ def test_zzz_logtostdout(self): def test_avail_easyconfig_params(self): """Test listing available easyconfig parameters.""" - def run_test(custom=None, extra_params=[]): + def run_test(custom=None, extra_params=[], fmt=None): """Inner function to run actual test in current setting.""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) - for avail_arg in [ - '-a', - '--avail-easyconfig-params', - ]: + avail_args = [ + '-a', + '--avail-easyconfig-params', + ] + for avail_arg in avail_args: # clear log write_file(self.logfile, '') args = [ - avail_arg, - '--unittest-file=%s' % self.logfile, - ] + '--unittest-file=%s' % self.logfile, + avail_arg, + ] + if fmt is not None: + args.append(fmt) if custom is not None: args.extend(['-e', custom]) @@ -358,17 +367,20 @@ def run_test(custom=None, extra_params=[]): par_types.append(CUSTOM) for param_type in [x[1] for x in par_types]: - self.assertTrue(re.search("%s\n%s" % (param_type.upper(), '-' * len(param_type)), outtxt), - "Parameter type %s is featured in output of eb %s (args: %s): %s" % - (param_type, avail_arg, args, outtxt)) + # regex for parameter group title, matches both txt and rst formats + regex = re.compile("%s.*\n%s" % (param_type, '-' * len(param_type)), re.I) + tup = (param_type, avail_arg, args, outtxt) + msg = "Parameter type %s is featured in output of eb %s (args: %s): %s" % tup + self.assertTrue(regex.search(outtxt), msg) # check a couple of easyconfig parameters for param in ["name", "version", "toolchain", "versionsuffix", "buildopts", "sources", "start_dir", "dependencies", "group", "exts_list", "moduleclass", "buildstats"] + extra_params: - self.assertTrue(re.search("%s(?:\(\*\))?:\s*\w.*" % param, outtxt), - "Parameter %s is listed with help in output of eb %s (args: %s): %s" % - (param, avail_arg, args, outtxt) - ) + # regex for parameter name (with optional '*') & description, matches both txt and rst formats + regex = re.compile("^[`]*%s(?:\*)?[`]*\s+\w+" % param, re.M) + tup = (param, avail_arg, args, regex.pattern, outtxt) + msg = "Parameter %s is listed with help in output of eb %s (args: %s, regex: %s): %s" % tup + self.assertTrue(regex.search(outtxt), msg) modify_env(os.environ, self.orig_environ) tempfile.tempdir = None @@ -376,9 +388,11 @@ def run_test(custom=None, extra_params=[]): if os.path.exists(dummylogfn): os.remove(dummylogfn) - run_test(custom='EB_foo', extra_params=['foo_extra1', 'foo_extra2']) - run_test(custom='bar', extra_params=['bar_extra1', 'bar_extra2']) - run_test(custom='EB_foofoo', extra_params=['foofoo_extra1', 'foofoo_extra2']) + for fmt in [None, 'txt', 'rst']: + run_test(fmt=fmt) + run_test(custom='EB_foo', extra_params=['foo_extra1', 'foo_extra2'], fmt=fmt) + run_test(custom='bar', extra_params=['bar_extra1', 'bar_extra2'], fmt=fmt) + run_test(custom='EB_foofoo', extra_params=['foofoo_extra1', 'foofoo_extra2'], fmt=fmt) # double underscore to make sure it runs first, which is required to detect certain types of bugs, # e.g. running with non-initialized EasyBuild config (truly mimicing 'eb --list-toolchains') @@ -447,6 +461,39 @@ def test_avail_lists(self): if os.path.exists(dummylogfn): os.remove(dummylogfn) + def test_avail_cfgfile_constants(self): + """Test --avail-cfgfile-constants.""" + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory + # to check whether easyconfigs install path is auto-included in robot path + tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') + mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) + + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + shutil.copytree(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) + + orig_sys_path = sys.path[:] + sys.path.insert(0, tmpdir) # prepend to give it preference over possible other installed easyconfigs pkgs + + args = [ + '--avail-cfgfile-constants', + '--unittest-file=%s' % self.logfile, + ] + outtxt = self.eb_main(args, logfile=dummylogfn) + cfgfile_constants = { + 'DEFAULT_ROBOT_PATHS': os.path.join(tmpdir, 'easybuild', 'easyconfigs'), + } + for cst_name, cst_value in cfgfile_constants.items(): + cst_regex = re.compile("^\*\s%s:\s.*\s\[value: .*%s.*\]" % (cst_name, cst_value), re.M) + tup = (cst_regex.pattern, outtxt) + self.assertTrue(cst_regex.search(outtxt), "Pattern '%s' in --avail-cfgfile_constants output: %s" % tup) + + if os.path.exists(dummylogfn): + os.remove(dummylogfn) + sys.path[:] = orig_sys_path + def test_list_easyblocks(self): """Test listing easyblock hierarchy.""" @@ -535,10 +582,11 @@ def test_search(self): args = [ search_arg, 'toy-0.0', - '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + '-r', + os.path.join(os.path.dirname(__file__), 'easyconfigs'), '--unittest-file=%s' % self.logfile, ] - outtxt = self.eb_main(args, logfile=dummylogfn) + outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True, verbose=True) info_msg = r"Searching \(case-insensitive\) for 'toy-0.0' in" self.assertTrue(re.search(info_msg, outtxt), "Info message when searching for easyconfigs in '%s'" % outtxt) @@ -550,16 +598,15 @@ def test_search(self): os.remove(dummylogfn) def test_dry_run(self): - """Test dry runs.""" - + """Test dry run (long format).""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) args = [ os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.4-GCC-4.6.3.eb'), - '--dry-run', - '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + '--dry-run', # implies enabling dependency resolution '--unittest-file=%s' % self.logfile, + '--robot-paths=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), ] outtxt = self.eb_main(args, logfile=dummylogfn) @@ -573,12 +620,29 @@ def test_dry_run(self): regex = re.compile(r" \* \[%s\] \S+%s \(module: %s\)" % (mark, ec, mod), re.M) self.assertTrue(regex.search(outtxt), "Found match for pattern %s in '%s'" % (regex.pattern, outtxt)) + def test_dry_run_short(self): + """Test dry run (short format).""" + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory + # to check whether easyconfigs install path is auto-included in robot path + tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') + mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) + + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + shutil.copytree(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) + + orig_sys_path = sys.path[:] + sys.path.insert(0, tmpdir) # prepend to give it preference over possible other installed easyconfigs pkgs + for dry_run_arg in ['-D', '--dry-run-short']: open(self.logfile, 'w').write('') args = [ - os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.4-GCC-4.6.3.eb'), + os.path.join(tmpdir, 'easybuild', 'easyconfigs', 'gzip-1.4-GCC-4.6.3.eb'), dry_run_arg, - '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + # purposely specifying senseless dir, to test auto-inclusion of easyconfigs pkg path in robot path + '--robot=%s' % os.path.join(tmpdir, 'robot_decoy'), '--unittest-file=%s' % self.logfile, ] outtxt = self.eb_main(args, logfile=dummylogfn) @@ -597,6 +661,10 @@ def test_dry_run(self): if os.path.exists(dummylogfn): os.remove(dummylogfn) + # cleanup + shutil.rmtree(tmpdir) + sys.path[:] = orig_sys_path + def test_try_robot_force(self): """ Test correct behavior for combination of --try-toolchain --robot --force. @@ -656,11 +724,8 @@ def test_dry_run_hierarchical(self): '--ignore-osdeps', '--force', '--debug', + '--robot-paths=%s' % os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs'), ] - errmsg = r"No robot path specified, which is required when looking for easyconfigs \(use --robot\)" - self.assertErrorRegex(EasyBuildError, errmsg, self.eb_main, args, logfile=dummylogfn, raise_error=True) - - args.append('--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs')) outtxt = self.eb_main(args, logfile=dummylogfn, verbose=True, raise_error=True) ecs_mods = [ @@ -691,8 +756,8 @@ def test_from_pr(self): tmpdir = tempfile.mkdtemp() args = [ - # PR for ictce/6.2.5, see https://github.com/hpcugent/easybuild-easyconfigs/pull/726/files - '--from-pr=726', + # PR for foss/2015a, see https://github.com/hpcugent/easybuild-easyconfigs/pull/1239/files + '--from-pr=1239', '--dry-run', # an argument must be specified to --robot, since easybuild-easyconfigs may not be installed '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), @@ -700,26 +765,88 @@ def test_from_pr(self): '--github-user=easybuild_test', # a GitHub token should be available for this user '--tmpdir=%s' % tmpdir, ] - outtxt = self.eb_main(args, logfile=dummylogfn, verbose=True) - - modules = [ - 'icc/2013_sp1.2.144', - 'ifort/2013_sp1.2.144', - 'impi/4.1.3.049', - 'imkl/11.1.2.144', - 'ictce/6.2.5', - 'gzip/1.6-ictce-6.2.5', - ] - for module in modules: - ec_fn = "%s.eb" % '-'.join(module.split('/')) - regex = re.compile(r"^ \* \[.\] .*/%s \(module: %s\)$" % (ec_fn, module), re.M) + try: + outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) + modules = [ + (tmpdir, 'FFTW/3.3.4-gompi-2015a'), + (tmpdir, 'foss/2015a'), + ('.*', 'GCC/4.9.2'), # not included in PR + (tmpdir, 'gompi/2015a'), + (tmpdir, 'HPL/2.1-foss-2015a'), + (tmpdir, 'hwloc/1.10.0-GCC-4.9.2'), + (tmpdir, 'numactl/2.0.10-GCC-4.9.2'), + (tmpdir, 'OpenBLAS/0.2.13-GCC-4.9.2-LAPACK-3.5.0'), + (tmpdir, 'OpenMPI/1.8.3-GCC-4.9.2'), + (tmpdir, 'OpenMPI/1.8.4-GCC-4.9.2'), + (tmpdir, 'ScaLAPACK/2.0.2-gompi-2015a-OpenBLAS-0.2.13-LAPACK-3.5.0'), + ] + for path_prefix, module in modules: + ec_fn = "%s.eb" % '-'.join(module.split('/')) + regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path_prefix, ec_fn, module), re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + + # make sure that *only* these modules are listed, no others + regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M) + self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules)) + + pr_tmpdir = os.path.join(tmpdir, 'easybuild-\S{6}', 'files_pr1239') + regex = re.compile("Prepended list of robot search paths with %s:" % pr_tmpdir, re.M) self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + except URLError, err: + print "Ignoring URLError '%s' in test_from_pr" % err + shutil.rmtree(tmpdir) + + def test_from_pr_listed_ecs(self): + """Test --from-pr in combination with specifying easyconfigs on the command line.""" + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) - pr_tmpdir = os.path.join(tmpdir, 'easybuild-\S{6}', 'files_pr726') - regex = re.compile("Prepended list of robot search paths with %s:" % pr_tmpdir, re.M) - self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory + test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + ecstmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') + mkdir(os.path.join(ecstmpdir, 'easybuild'), parents=True) + shutil.copytree(test_ecs_path, os.path.join(ecstmpdir, 'easybuild', 'easyconfigs')) - shutil.rmtree(tmpdir) + # inject path to test easyconfigs into head of Python search path + sys.path.insert(0, ecstmpdir) + + tmpdir = tempfile.mkdtemp() + args = [ + 'toy-0.0.eb', + 'gompi-2015a.eb', # also pulls in GCC, OpenMPI (which pulls in hwloc and numactl) + 'GCC-4.6.3.eb', + # PR for foss/2015a, see https://github.com/hpcugent/easybuild-easyconfigs/pull/1239/files + '--from-pr=1239', + '--dry-run', + # an argument must be specified to --robot, since easybuild-easyconfigs may not be installed + '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + '--unittest-file=%s' % self.logfile, + '--github-user=easybuild_test', # a GitHub token should be available for this user + '--tmpdir=%s' % tmpdir, + ] + try: + outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) + modules = [ + (ecstmpdir, 'toy/0.0'), + ('.*', 'GCC/4.9.2'), # not included in PR + (tmpdir, 'hwloc/1.10.0-GCC-4.9.2'), + (tmpdir, 'numactl/2.0.10-GCC-4.9.2'), + (tmpdir, 'OpenMPI/1.8.4-GCC-4.9.2'), + (tmpdir, 'gompi/2015a'), + ('.*', 'GCC/4.6.3'), + ] + for path_prefix, module in modules: + ec_fn = "%s.eb" % '-'.join(module.split('/')) + regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path_prefix, ec_fn, module), re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + + # make sure that *only* these modules are listed, no others + regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M) + self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules)) + + except URLError, err: + print "Ignoring URLError '%s' in test_from_pr" % err + shutil.rmtree(tmpdir) def test_no_such_software(self): """Test using no arguments.""" @@ -838,6 +965,7 @@ def test_tmpdir(self): def test_ignore_osdeps(self): """Test ignoring of listed OS dependencies.""" txt = '\n'.join([ + 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', @@ -918,6 +1046,10 @@ def test_experimental(self): def test_deprecated(self): """Test the deprecated option""" + if 'EASYBUILD_DEPRECATED' in os.environ: + os.environ['EASYBUILD_DEPRECATED'] = str(VERSION) + init_config() + orig_value = easybuild.tools.build_log.CURRENT_VERSION # make sure it's off by default @@ -945,7 +1077,7 @@ def test_deprecated(self): except easybuild.tools.build_log.EasyBuildError, err2: self.assertTrue('DEPRECATED' in str(err2)) - # force higher version by prefixing it with 1, which should result in deprecation + # force higher version by prefixing it with 1, which should result in deprecation errors topt = EasyBuildOptions( go_args=['--deprecated=1%s' % orig_value], ) @@ -1012,13 +1144,20 @@ def test_allow_modules_tool_mismatch(self): def test_try(self): """Test whether --try options are taken into account.""" - ec_file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'toy-0.0.eb') + ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + tweaked_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb') + shutil.copy2(os.path.join(ecs_path, 'toy-0.0.eb'), tweaked_toy_ec) + f = open(tweaked_toy_ec, 'a') + f.write("easyblock = 'ConfigureMake'") + f.close() + args = [ - ec_file, + tweaked_toy_ec, '--sourcepath=%s' % self.test_sourcepath, '--buildpath=%s' % self.test_buildpath, '--installpath=%s' % self.test_installpath, '--dry-run', + '--robot=%s' % ecs_path, ] test_cases = [ @@ -1029,6 +1168,14 @@ def test_try(self): (['--try-toolchain-name=gompi', '--try-toolchain-version=1.4.10'], 'toy/0.0-gompi-1.4.10'), (['--try-software-version=1.2.3', '--try-toolchain=gompi,1.4.10'], 'toy/1.2.3-gompi-1.4.10'), (['--try-amend=versionsuffix=-test'], 'toy/0.0-test'), + # tweak existing list-typed value (patches) + (['--try-amend=versionsuffix=-test2', '--try-amend=patches=1.patch,2.patch'], 'toy/0.0-test2'), + # append to existing list-typed value (patches) + (['--try-amend=versionsuffix=-test3', '--try-amend=patches=,extra.patch'], 'toy/0.0-test3'), + # prepend to existing list-typed value (patches) + (['--try-amend=versionsuffix=-test4', '--try-amend=patches=extra.patch,'], 'toy/0.0-test4'), + # define extra list-typed parameter + (['--try-amend=versionsuffix=-test5', '--try-amend=exts_list=1,2,3'], 'toy/0.0-test5'), # only --try causes other build specs to be included too (['--try-software=foo,1.2.3', '--toolchain=gompi,1.4.10'], 'foo/1.2.3-gompi-1.4.10'), (['--software=foo,1.2.3', '--try-toolchain=gompi,1.4.10'], 'foo/1.2.3-gompi-1.4.10'), @@ -1054,7 +1201,7 @@ def test_recursive_try(self): tweaked_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb') shutil.copy2(os.path.join(ecs_path, 'toy-0.0.eb'), tweaked_toy_ec) f = open(tweaked_toy_ec, 'a') - f.write("dependencies = [('gzip', '1.4')]") # add fictious dependency + f.write("dependencies = [('gzip', '1.4')]\n") # add fictious dependency f.close() sourcepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox', 'sources') @@ -1092,13 +1239,18 @@ def test_recursive_try(self): #mod_regex = re.compile("%s \(module: .*%s\)$" % (ec, mod), re.M) self.assertTrue(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) + # clear fictious dependency + f = open(tweaked_toy_ec, 'a') + f.write("dependencies = []\n") + f.close() + # no recursive try if --(try-)software(-X) is involved for extra_args in [['--try-software-version=1.2.3'], ['--software-version=1.2.3']]: outtxt = self.eb_main(args + extra_args, raise_error=True) - for mod in ['toy/1.2.3-gompi-1.4.10', 'gzip/1.4-gompi-1.4.10', 'gompi/1.4.10', 'GCC/4.7.2']: + for mod in ['toy/1.2.3-gompi-1.4.10', 'gompi/1.4.10', 'GCC/4.7.2']: mod_regex = re.compile("\(module: %s\)$" % mod, re.M) self.assertTrue(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) - for mod in ['gzip/1.2.3-gompi-1.4.10']: + for mod in ['gompi/1.2.3', 'GCC/1.2.3']: mod_regex = re.compile("\(module: %s\)$" % mod, re.M) self.assertFalse(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) @@ -1125,7 +1277,7 @@ def test_cleanup_builddir(self): args = [ toy_ec, '--force', - '--try-amend=premakeopts=nosuchcommand &&', + '--try-amend=prebuildopts=nosuchcommand &&', ] self.eb_main(args, do_build=True) self.assertTrue(os.path.exists(toy_buildpath), "Build dir %s is retained after failed build" % toy_buildpath) @@ -1206,6 +1358,61 @@ def toy(extra_args=None): tup = (filter_arg_regex.pattern, test_report_txt) self.assertTrue(filter_arg_regex.search(test_report_txt), "%s in %s" % tup) + def test_robot(self): + """Test --robot and --robot-paths command line options.""" + test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + eb_file = os.path.join(test_ecs_path, 'gzip-1.4-GCC-4.6.3.eb') # includes 'toy/.0.0-deps' as a dependency + + # hide test modules + self.reset_modulepath([]) + + # dependency resolution is disabled by default, even if required paths are available + args = [ + eb_file, + '--robot-paths=%s' % test_ecs_path, + ] + error_regex ='no module .* found for dependency' + self.assertErrorRegex(EasyBuildError, error_regex, self.eb_main, args, raise_error=True, do_build=True) + + # enable robot, but without passing path required to resolve toy dependency => FAIL + args = [ + eb_file, + '--robot', + '--dry-run', + ] + self.assertErrorRegex(EasyBuildError, 'Irresolvable dependencies', self.eb_main, args, raise_error=True) + + # add path to test easyconfigs to robot paths, so dependencies can be resolved + self.eb_main(args + ['--robot-paths=%s' % test_ecs_path], raise_error=True) + + # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory + # to check whether easyconfigs install path is auto-included in robot path + tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') + mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) + shutil.copytree(test_ecs_path, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) + + # prepend path to test easyconfigs into Python search path, so it gets picked up as --robot-paths default + orig_sys_path = sys.path[:] + sys.path.insert(0, tmpdir) + self.eb_main(args, raise_error=True) + + shutil.rmtree(tmpdir) + sys.path[:] = orig_sys_path + + # make sure that paths specified to --robot get preference over --robot-paths + args = [ + eb_file, + '--robot=%s' % test_ecs_path, + '--robot-paths=%s' % os.path.join(tmpdir, 'easybuild', 'easyconfigs'), + '--dry-run', + ] + outtxt = self.eb_main(args, raise_error=True) + + for ec in ['GCC-4.6.3.eb', 'ictce-4.1.13.eb', 'toy-0.0-deps.eb', 'gzip-1.4-GCC-4.6.3.eb']: + ec_regex = re.compile('^\s\*\s\[[xF ]\]\s%s' % os.path.join(test_ecs_path, ec), re.M) + self.assertTrue(ec_regex.search(outtxt), "Pattern %s found in %s" % (ec_regex.pattern, outtxt)) + + def suite(): """ returns all the testcases in this module """ return TestLoader().loadTestsFromTestCase(CommandLineOptionsTest) diff --git a/test/framework/parallelbuild.py b/test/framework/parallelbuild.py index 395b4c127a..637d56061e 100644 --- a/test/framework/parallelbuild.py +++ b/test/framework/parallelbuild.py @@ -32,9 +32,10 @@ from unittest import TestLoader, main from vsc.utils.fancylogger import setLogLevelDebug, logToScreen -from easybuild.framework.easyconfig.tools import process_easyconfig, resolve_dependencies +from easybuild.framework.easyconfig.tools import process_easyconfig from easybuild.tools import config, parallelbuild from easybuild.tools.parallelbuild import PbsJob, build_easyconfigs_in_parallel +from easybuild.tools.robot import resolve_dependencies def mock(*args, **kwargs): diff --git a/test/framework/robot.py b/test/framework/robot.py index 6b83103e42..c157a25fad 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -35,13 +35,16 @@ from unittest import main as unittestmain import easybuild.framework.easyconfig.tools as ectools -from easybuild.framework.easyconfig.tools import resolve_dependencies, skip_available +import easybuild.tools.robot as robot +from easybuild.framework.easyconfig.tools import skip_available from easybuild.tools import config, modules from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.robot import resolve_dependencies from test.framework.utilities import find_full_path ORIG_MODULES_TOOL = modules.modules_tool -ORIG_MAIN_MODULES_TOOL = ectools.modules_tool +ORIG_ECTOOLS_MODULES_TOOL = ectools.modules_tool +ORIG_ROBOT_MODULES_TOOL = robot.modules_tool ORIG_MODULE_FUNCTION = os.environ.get('module', None) @@ -60,8 +63,8 @@ def available(self, *args): return self.avail_modules def show(self, modname): - """Dummy implementation of show, which includes full path to (existing) module files.""" - if modname in self.avail_modules: + """Dummy implementation of show, which includes full path to (available or hidden) module files.""" + if modname in self.avail_modules or os.path.basename(modname).startswith('.'): txt = ' %s:' % os.path.join('/tmp', modname) else: txt = 'Module %s not found' % modname @@ -82,6 +85,7 @@ def setUp(self): # replace Modules class with something we have control over config.modules_tool = mock_module ectools.modules_tool = mock_module + robot.modules_tool = mock_module os.environ['module'] = "() { eval `/bin/echo $*`\n}" self.base_easyconfig_dir = find_full_path(os.path.join("test", "framework", "easyconfigs")) @@ -120,10 +124,11 @@ def test_resolve_dependencies(self): 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, + 'hidden': False, }], 'parsed': True, } - build_options.update({'robot_path': self.base_easyconfig_dir}) + build_options.update({'robot': True, 'robot_path': self.base_easyconfig_dir}) init_config(build_options=build_options) res = resolve_dependencies([deepcopy(easyconfig_dep)]) # dependency should be found, order should be correct @@ -131,7 +136,7 @@ def test_resolve_dependencies(self): self.assertEqual('gzip/1.4', res[0]['full_mod_name']) self.assertEqual('foo/1.2.3', res[-1]['full_mod_name']) - # hidden dependencies are found too + # hidden dependencies are found too, but only retained if they're not available (or forced to be retained hidden_dep = { 'name': 'toy', 'version': '0.0', @@ -143,11 +148,20 @@ def test_resolve_dependencies(self): easyconfig_moredeps = deepcopy(easyconfig_dep) easyconfig_moredeps['dependencies'].append(hidden_dep) easyconfig_moredeps['hiddendependencies'] = [hidden_dep] + + # toy/.0.0-deps is available and thus should be omitted res = resolve_dependencies([deepcopy(easyconfig_moredeps)]) + self.assertEqual(len(res), 2) + full_mod_names = [ec['full_mod_name'] for ec in res] + self.assertFalse('toy/.0.0-deps' in full_mod_names) + + res = resolve_dependencies([deepcopy(easyconfig_moredeps)], retain_all_deps=True) self.assertEqual(len(res), 4) # hidden dep toy/.0.0-deps (+1) depends on (fake) ictce/4.1.13 (+1) self.assertEqual('gzip/1.4', res[0]['full_mod_name']) self.assertEqual('foo/1.2.3', res[-1]['full_mod_name']) - self.assertTrue('toy/.0.0-deps' in [ec['full_mod_name'] for ec in res]) + full_mod_names = [ec['full_mod_name'] for ec in res] + self.assertTrue('toy/.0.0-deps' in full_mod_names) + self.assertTrue('ictce/4.1.13' in full_mod_names) # here we have included a dependency in the easyconfig list easyconfig['full_mod_name'] = 'gzip/1.4' @@ -171,6 +185,7 @@ def test_resolve_dependencies(self): 'versionsuffix': '', 'toolchain': {'name': 'GCC', 'version': '4.6.3'}, 'dummy': True, + 'hidden': False, }] ecs = [deepcopy(easyconfig_dep)] build_options.update({'robot_path': self.base_easyconfig_dir}) @@ -197,6 +212,7 @@ def test_resolve_dependencies(self): 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, + 'hidden': False, }] ecs = [deepcopy(easyconfig_dep)] res = resolve_dependencies(ecs) @@ -225,7 +241,7 @@ def test_resolve_dependencies(self): # build that are listed but already have a module available are not retained without force build_options.update({'force': False}) init_config(build_options=build_options) - newecs = skip_available(ecs, testing=True) # skip available builds since force is not enabled + newecs = skip_available(ecs) # skip available builds since force is not enabled res = resolve_dependencies(newecs) self.assertEqual(len(res), 2) self.assertEqual('goolf/1.4.10', res[0]['full_mod_name']) @@ -235,7 +251,7 @@ def test_resolve_dependencies(self): build_options.update({'retain_all_deps': True}) init_config(build_options=build_options) ecs = [deepcopy(easyconfig_dep)] - newecs = skip_available(ecs, testing=True) # skip available builds since force is not enabled + newecs = skip_available(ecs) # skip available builds since force is not enabled res = resolve_dependencies(newecs) self.assertEqual(len(res), 9) self.assertEqual('GCC/4.7.2', res[0]['full_mod_name']) @@ -259,6 +275,7 @@ def test_resolve_dependencies(self): 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, + 'hidden': False, }] ecs = [deepcopy(easyconfig_dep)] res = resolve_dependencies([deepcopy(easyconfig_dep)]) @@ -276,7 +293,8 @@ def tearDown(self): super(RobotTest, self).tearDown() config.modules_tool = ORIG_MODULES_TOOL - ectools.modules_tool = ORIG_MAIN_MODULES_TOOL + ectools.modules_tool = ORIG_ECTOOLS_MODULES_TOOL + robot.modules_tool = ORIG_ROBOT_MODULES_TOOL if ORIG_MODULE_FUNCTION is not None: os.environ['module'] = ORIG_MODULE_FUNCTION else: diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/bar.py b/test/framework/sandbox/easybuild/easyblocks/generic/bar.py index 482ee15cc1..2dcc4c5c01 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/bar.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/bar.py @@ -38,9 +38,8 @@ class bar(EasyBlock): @staticmethod def extra_options(): """Custom easyconfig parameters for bar.""" - - extra_vars = [ - ('bar_extra1', [None, "first bar-specific easyconfig parameter (mandatory)", MANDATORY]), - ('bar_extra2', ['BAR', "second bar-specific easyconfig parameter", CUSTOM]), - ] + extra_vars = { + 'bar_extra1': [None, "first bar-specific easyconfig parameter (mandatory)", MANDATORY], + 'bar_extra2': ['BAR', "second bar-specific easyconfig parameter", CUSTOM], + } return EasyBlock.extra_options(extra_vars) diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py new file mode 100644 index 0000000000..8024520f32 --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py @@ -0,0 +1,34 @@ +## +# Copyright 2009-2014 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for building and installing dummy extensions, implemented as an easyblock + +@author: Kenneth Hoste (Ghent University) +""" + +from easybuild.framework.extensioneasyblock import ExtensionEasyBlock + +class DummyExtension(ExtensionEasyBlock): + """Support for building/installing dummy extensions.""" diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py b/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py index db67440c90..7ba202cbdc 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py @@ -31,4 +31,11 @@ class Toolchain(EasyBlock): """Dummy support for toolchains.""" - pass + def configure_step(self): + pass + def build_step(self): + pass + def install_step(self): + pass + def sanity_check_step(self): + pass diff --git a/test/framework/sandbox/easybuild/easyblocks/hpl.py b/test/framework/sandbox/easybuild/easyblocks/hpl.py new file mode 100644 index 0000000000..1ef029f6c1 --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/hpl.py @@ -0,0 +1,33 @@ +## +# Copyright 2009-2014 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Dummy easyblock for HPL + +@author: Kenneth Hoste (Ghent University) +""" +from easybuild.framework.easyblock import EasyBlock + +class EB_HPL(EasyBlock): + pass diff --git a/test/framework/sandbox/easybuild/easyblocks/scalapack.py b/test/framework/sandbox/easybuild/easyblocks/scalapack.py new file mode 100644 index 0000000000..4534a5cd10 --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/scalapack.py @@ -0,0 +1,33 @@ +## +# Copyright 2009-2014 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Dummy easyblock for ScaLAPACK + +@author: Kenneth Hoste (Ghent University) +""" +from easybuild.framework.easyblock import EasyBlock + +class EB_ScaLAPACK(EasyBlock): + pass diff --git a/test/framework/sandbox/easybuild/easyblocks/toy.py b/test/framework/sandbox/easybuild/easyblocks/toy.py index 897d248dfe..b03b6d9959 100644 --- a/test/framework/sandbox/easybuild/easyblocks/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/toy.py @@ -33,8 +33,9 @@ import shutil from easybuild.framework.easyblock import EasyBlock -from easybuild.tools.filetools import mkdir, run_cmd +from easybuild.tools.filetools import mkdir from easybuild.tools.modules import get_software_root, get_software_version +from easybuild.tools.run import run_cmd class EB_toy(EasyBlock): """Support for building/installing toy.""" @@ -61,9 +62,9 @@ def build_step(self, name=None): """Build toy.""" if name is None: name = self.name - run_cmd('%(premakeopts)s gcc %(name)s.c -o %(name)s' % { + run_cmd('%(prebuildopts)s gcc %(name)s.c -o %(name)s' % { 'name': name, - 'premakeopts': self.cfg['premakeopts'], + 'prebuildopts': self.cfg['prebuildopts'], }) def install_step(self, name=None): diff --git a/test/framework/sandbox/sources/g/gzip/gzip-1.4.tar.gz b/test/framework/sandbox/sources/g/gzip/gzip-1.4.tar.gz new file mode 100644 index 0000000000..92d6ae4e08 Binary files /dev/null and b/test/framework/sandbox/sources/g/gzip/gzip-1.4.tar.gz differ diff --git a/test/framework/scripts.py b/test/framework/scripts.py index f9704b7937..1776f8cdc1 100644 --- a/test/framework/scripts.py +++ b/test/framework/scripts.py @@ -65,13 +65,14 @@ def test_generate_software_list(self): out, ec = run_cmd(cmd, simple=False) # make sure output is kind of what we expect it to be - regex = r"Supported Packages \(12 " + regex = r"Supported Packages \(18 " self.assertTrue(re.search(regex, out), "Pattern '%s' found in output: %s" % (regex, out)) per_letter = { + 'C': '1', # CUDA 'F': '1', # FFTW 'G': '4', # GCC, gompi, goolf, gzip 'H': '1', # hwloc - 'I': '2', # ictce, impi + 'I': '7', # icc, iccifort, ictce, ifort, iimpi, imkl, impi 'O': '2', # OpenMPI, OpenBLAS 'S': '1', # ScaLAPACK 'T': '1', # toy diff --git a/test/framework/suite.py b/test/framework/suite.py index 61b3ac30b4..1ed29b598a 100644 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -69,6 +69,7 @@ import test.framework.toolchain as tc import test.framework.toolchainvariables as tcv import test.framework.toy_build as t +import test.framework.tweak as tw import test.framework.variables as v @@ -95,7 +96,7 @@ # call suite() for each module and then run them all # note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config -tests = [o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, l, f_c, sc] +tests = [o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, l, f_c, sc, tw] SUITE = unittest.TestSuite([x.suite() for x in tests]) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 8f9e8a4cf2..8447913dc7 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -27,7 +27,8 @@ @author: Kenneth hoste (Ghent University) """ -from test.framework.utilities import EnhancedTestCase +import os +from test.framework.utilities import EnhancedTestCase, init_config from unittest import TestLoader, main from easybuild.tools.systemtools import AMD, ARM, DARWIN, INTEL, LINUX, UNKNOWN @@ -40,11 +41,11 @@ class SystemToolsTest(EnhancedTestCase): """ very basis FileRepository test, we don't want git / svn dependency """ - def test_core_count(self): + def test_avail_core_count(self): """Test getting core count.""" - for core_count in [get_avail_core_count(), get_core_count()]: - self.assertTrue(isinstance(core_count, int), "core_count has type int: %s, %s" % (core_count, type(core_count))) - self.assertTrue(core_count > 0, "core_count %d > 0" % core_count) + core_count = get_avail_core_count() + self.assertTrue(isinstance(core_count, int), "core_count has type int: %s, %s" % (core_count, type(core_count))) + self.assertTrue(core_count > 0, "core_count %d > 0" % core_count) def test_cpu_model(self): """Test getting CPU model.""" diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 6d76ac1100..dc60b2c041 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -38,6 +38,7 @@ import easybuild.tools.modules as modules from easybuild.framework.easyconfig.easyconfig import EasyConfig, ActiveMNS from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import write_file from easybuild.tools.toolchain.utilities import search_toolchain from test.framework.utilities import find_full_path @@ -408,8 +409,8 @@ def test_goolfc(self): # check CUDA runtime lib self.assertTrue("-lrt -lcudart" in tc.get_variable('LIBS')) - def test_ictce_toolchain(self): - """Test for ictce toolchain.""" + def setup_sandbox_for_intel_fftw(self): + """Set up sandbox for Intel FFTW""" # hack to make Intel FFTW lib check pass # rewrite $root in imkl module so we can put required lib*.a files in place tmpdir = tempfile.mkdtemp() @@ -426,7 +427,13 @@ def test_ictce_toolchain(self): for subdir in ['mkl/lib/intel64', 'compiler/lib/intel64']: os.makedirs(os.path.join(tmpdir, subdir)) for fftlib in fftw_libs: - open(os.path.join(tmpdir, subdir, 'lib%s.a' % fftlib), 'w').write('foo') + write_file(os.path.join(tmpdir, subdir, 'lib%s.a' % fftlib), 'foo') + + return tmpdir, imkl_module_path, imkl_module_txt + + def test_ictce_toolchain(self): + """Test for ictce toolchain.""" + tmpdir, imkl_module_path, imkl_module_txt = self.setup_sandbox_for_intel_fftw() tc = self.get_toolchain("ictce", version="4.1.13") tc.prepare() @@ -496,6 +503,25 @@ def test_toolchain_verification(self): tc.set_options(opts) tc.prepare() + def test_nosuchtoolchain(self): + """Test preparing for a toolchain for which no module is available.""" + tc = self.get_toolchain('intel', version='1970.01') + self.assertErrorRegex(EasyBuildError, "No module found for toolchain", tc.prepare) + + def test_mpi_cmd_for(self): + """Test mpi_cmd_for function.""" + tmpdir, imkl_module_path, imkl_module_txt = self.setup_sandbox_for_intel_fftw() + + tc = self.get_toolchain('ictce', version='4.1.13') + tc.prepare() + + mpi_cmd_for_re = re.compile("^mpirun --file=.*/mpdboot -machinefile .*/nodes -np 4 test$") + self.assertTrue(mpi_cmd_for_re.match(tc.mpi_cmd_for('test', 4))) + + # cleanup + shutil.rmtree(tmpdir) + open(imkl_module_path, 'w').write(imkl_module_txt) + def tearDown(self): """Cleanup.""" # purge any loaded modules before restoring $MODULEPATH diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 09b16d6ad9..95eb21b03b 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -57,17 +57,6 @@ def setUp(self): fd, self.dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) - # adjust PYTHONPATH such that test easyblocks are found - import easybuild - eb_blocks_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'sandbox')) - if not eb_blocks_path in sys.path: - sys.path.append(eb_blocks_path) - easybuild = reload(easybuild) - - import easybuild.easyblocks - reload(easybuild.easyblocks) - reload(easybuild.tools.module_naming_scheme) - # clear log write_file(self.logfile, '') @@ -519,18 +508,19 @@ def test_allow_system_deps(self): def test_toy_hierarchical(self): """Test toy build under example hierarchical module naming scheme.""" + test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') self.setup_hierarchical_modules() mod_prefix = os.path.join(self.test_installpath, 'modules', 'all') args = [ - os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb'), + os.path.join(test_easyconfigs, 'toy-0.0.eb'), '--sourcepath=%s' % self.test_sourcepath, '--buildpath=%s' % self.test_buildpath, '--installpath=%s' % self.test_installpath, '--debug', '--unittest-file=%s' % self.logfile, '--force', - '--robot=%s' % os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs'), + '--robot=%s' % test_easyconfigs, '--module-naming-scheme=HierarchicalMNS', ] @@ -625,13 +615,15 @@ def test_toy_hierarchical(self): # no dependencies or toolchain => no module load statements in module file modtxt = read_file(toy_module_path) modpath_extension = os.path.join(mod_prefix, 'Compiler', 'toy', '0.0') - self.assertTrue(re.search("^module\s*use\s*%s" % modpath_extension, modtxt, re.M)) + self.assertTrue(re.search(r"^module\s*use\s*%s" % modpath_extension, modtxt, re.M)) os.remove(toy_module_path) # building a toolchain module should also work - args = ['gompi-1.4.10.eb'] + args[1:] + args[0] = os.path.join(test_easyconfigs, 'gompi-1.4.10.eb') modules_tool().purge() - self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True) + self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=False) + gompi_module_path = os.path.join(mod_prefix, 'Core', 'gompi', '1.4.10') + self.assertTrue(os.path.exists(gompi_module_path)) def test_toy_advanced(self): """Test toy build with extensions and non-dummy toolchain.""" diff --git a/test/framework/tweak.py b/test/framework/tweak.py new file mode 100644 index 0000000000..0395bd7d11 --- /dev/null +++ b/test/framework/tweak.py @@ -0,0 +1,116 @@ +## +# Copyright 2014 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Unit tests for framework/easyconfig/tweak.py + +@author: Kenneth Hoste (Ghent University) +""" +import os +from test.framework.utilities import EnhancedTestCase +from unittest import TestLoader, main + +from easybuild.framework.easyconfig.tweak import find_matching_easyconfigs, obtain_ec_for, pick_version + + +class TweakTest(EnhancedTestCase): + """Tests for tweak functionality.""" + def test_pick_version(self): + """Test pick_version function.""" + # if required version is not available, the most recent version less than or equal should be returned + self.assertEqual(('1.4', '1.0'), pick_version('1.4', ['0.5', '1.0', '1.5'])) + + # if required version is available, that should be what's returned + self.assertEqual(('1.0', '1.0'), pick_version('1.0', ['0.5', '1.0', '1.5'])) + + def test_find_matching_easyconfigs(self): + """Test find_matching_easyconfigs function.""" + test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + for (name, installver) in [('GCC', '4.8.2'), ('gzip', '1.5-goolf-1.4.10')]: + ecs = find_matching_easyconfigs(name, installver, [test_easyconfigs_path]) + self.assertTrue(len(ecs) == 1 and ecs[0].endswith('/%s-%s.eb' % (name, installver))) + + ecs = find_matching_easyconfigs('GCC', '*', [test_easyconfigs_path]) + gccvers = ['4.6.3', '4.6.4', '4.7.2', '4.8.2', '4.8.3', '4.9.2'] + self.assertEqual(len(ecs), len(gccvers)) + ecs_basename = [os.path.basename(ec) for ec in ecs] + for gccver in gccvers: + gcc_ec = 'GCC-%s.eb' % gccver + self.assertTrue(gcc_ec in ecs_basename, "%s is included in %s" % (gcc_ec, ecs_basename)) + + def test_obtain_ec_for(self): + """Test obtain_ec_for function.""" + test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + # find existing easyconfigs + specs = { + 'name': 'GCC', + 'version': '4.6.4', + } + (generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path]) + self.assertFalse(generated) + self.assertEqual(os.path.basename(ec_file), 'GCC-4.6.4.eb') + + specs = { + 'name': 'ScaLAPACK', + 'version': '2.0.2', + 'toolchain_name': 'gompi', + 'toolchain_version': '1.4.10', + 'versionsuffix': '-OpenBLAS-0.2.6-LAPACK-3.4.2', + } + (generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path]) + self.assertFalse(generated) + self.assertEqual(os.path.basename(ec_file), 'ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb') + + specs = { + 'name': 'ifort', + 'versionsuffix': '-GCC-4.8.3', + } + (generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path]) + self.assertFalse(generated) + self.assertEqual(os.path.basename(ec_file), 'ifort-2013.5.192-GCC-4.8.3.eb') + + # latest version if not specified + specs = { + 'name': 'GCC', + } + (generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path]) + self.assertFalse(generated) + self.assertEqual(os.path.basename(ec_file), 'GCC-4.9.2.eb') + + # generate non-existing easyconfig + specs = { + 'name': 'GCC', + 'version': '5.4.3', + } + (generated, ec_file) = obtain_ec_for(specs, [test_easyconfigs_path]) + self.assertTrue(generated) + self.assertEqual(os.path.basename(ec_file), 'GCC-5.4.3.eb') + + +def suite(): + """ return all the tests in this file """ + return TestLoader().loadTestsFromTestCase(TweakTest) + +if __name__ == '__main__': + main() diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 0aa37bdd63..f2b26c85e4 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -38,6 +38,7 @@ from vsc.utils import fancylogger from vsc.utils.patterns import Singleton +import easybuild.tools.build_log as eb_build_log import easybuild.tools.options as eboptions import easybuild.tools.toolchain.utilities as tc_utils import easybuild.tools.module_naming_scheme.toolchain as mns_toolchain @@ -105,6 +106,12 @@ def setUp(self): os.environ['EASYBUILD_BUILDPATH'] = self.test_buildpath self.test_installpath = tempfile.mkdtemp() os.environ['EASYBUILD_INSTALLPATH'] = self.test_installpath + + # make sure no deprecated behaviour is being triggered (unless intended by the test) + # trip *all* log.deprecated statements by setting deprecation version ridiculously high + self.orig_current_version = eb_build_log.CURRENT_VERSION + os.environ['EASYBUILD_DEPRECATED'] = '10000000' + init_config() # add test easyblocks to Python search path and (re)import and reload easybuild modules @@ -131,9 +138,13 @@ def tearDown(self): # restore original Python search path sys.path = self.orig_sys_path - for path in [self.test_buildpath, self.test_installpath, self.test_prefix]: + # cleanup + for path in [self.logfile, self.test_buildpath, self.test_installpath, self.test_prefix]: try: - shutil.rmtree(path) + if os.path.isdir(path): + shutil.rmtree(path) + else: + os.remove(path) except OSError, err: pass @@ -202,13 +213,17 @@ def setup_hierarchical_modules(self): # make sure only modules in a hierarchical scheme are available, mixing modules installed with # a flat scheme like EasyBuildMNS and a hierarhical one like HierarchicalMNS doesn't work - self.reset_modulepath([os.path.join(mod_prefix, 'Core')]) + self.reset_modulepath([mod_prefix, os.path.join(mod_prefix, 'Core')]) - # tweak use statements in GCC/OpenMPI modules to ensure correct paths + # tweak use statements in modules to ensure correct paths mpi_pref = os.path.join(mod_prefix, 'MPI', 'GCC', '4.7.2', 'OpenMPI', '1.6.4') for modfile in [ os.path.join(mod_prefix, 'Core', 'GCC', '4.7.2'), + os.path.join(mod_prefix, 'Core', 'GCC', '4.8.3'), + os.path.join(mod_prefix, 'Core', 'icc', '2013.5.192-GCC-4.8.3'), + os.path.join(mod_prefix, 'Core', 'ifort', '2013.5.192-GCC-4.8.3'), os.path.join(mod_prefix, 'Compiler', 'GCC', '4.7.2', 'OpenMPI', '1.6.4'), + os.path.join(mod_prefix, 'Compiler', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049'), os.path.join(mpi_pref, 'FFTW', '3.3.3'), os.path.join(mpi_pref, 'OpenBLAS', '0.2.6-LAPACK-3.4.2'), os.path.join(mpi_pref, 'ScaLAPACK', '2.0.2-OpenBLAS-0.2.6-LAPACK-3.4.2'), @@ -228,6 +243,7 @@ def cleanup(): # empty caches tc_utils._initial_toolchain_instances.clear() easyconfig._easyconfigs_cache.clear() + easyconfig._easyconfig_files_cache.clear() mns_toolchain._toolchain_details_cache.clear() @@ -248,7 +264,7 @@ def init_config(args=None, build_options=None): } if 'suffix_modules_path' not in build_options: build_options.update({'suffix_modules_path': GENERAL_CLASS}) - config.init_build_options(build_options) + config.init_build_options(build_options=build_options) return eb_go.options diff --git a/vsc/README.md b/vsc/README.md index c2fac06881..165e0109fb 100644 --- a/vsc/README.md +++ b/vsc/README.md @@ -1,3 +1,3 @@ Code from https://github.com/hpcugent/vsc-base -based on 95c2174a243874227dcc895d3e26c1b3b949ba22 (vsc-base v1.9.5) +based on eb47bee435e5e24666b398d8dd41f82a40214b7a (vsc-base v2.0.0) diff --git a/vsc/utils/fancylogger.py b/vsc/utils/fancylogger.py index b9576bbd68..1f0220df75 100644 --- a/vsc/utils/fancylogger.py +++ b/vsc/utils/fancylogger.py @@ -324,7 +324,7 @@ def thread_name(): return threading.currentThread().getName() -def getLogger(name=None, fname=True, clsname=False, fancyrecord=None): +def getLogger(name=None, fname=False, clsname=False, fancyrecord=None): """ returns a fancylogger if fname is True, the loggers name will be 'name[.classname].functionname' @@ -352,8 +352,10 @@ def getLogger(name=None, fname=True, clsname=False, fancyrecord=None): if os.environ.get('FANCYLOGGER_GETLOGGER_DEBUG', '0').lower() in ('1', 'yes', 'true', 'y'): print 'FANCYLOGGER_GETLOGGER_DEBUG', print 'name', name, 'fname', fname, 'fullname', fullname, - print 'parent_info verbose' - print "\n".join(l.get_parent_info("FANCYLOGGER_GETLOGGER_DEBUG")) + print "getRootLoggerName: ", getRootLoggerName() + if hasattr(l, 'get_parent_info'): + print 'parent_info verbose' + print "\n".join(l.get_parent_info("FANCYLOGGER_GETLOGGER_DEBUG")) sys.stdout.flush() return l @@ -683,8 +685,8 @@ def enableDefaultHandlers(): def getDetailsLogLevels(fancy=True): """ Return list of (name,loglevelname) pairs of existing loggers - - @param fancy: if True, returns only Fancylogger; if False, returns non-FancyLoggers, + + @param fancy: if True, returns only Fancylogger; if False, returns non-FancyLoggers, anything else, return all loggers """ func_map = { diff --git a/vsc/utils/generaloption.py b/vsc/utils/generaloption.py index 2dd49ed8ce..af6d6397c0 100644 --- a/vsc/utils/generaloption.py +++ b/vsc/utils/generaloption.py @@ -1,6 +1,6 @@ # # -# Copyright 2011-2013 Ghent University +# Copyright 2011-2014 Ghent University # # This file is part of vsc-base, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -72,20 +72,38 @@ def set_columns(cols=None): os.environ['COLUMNS'] = "%s" % cols +def what_str_list_tuple(name): + """Given name, return separator, class and helptext wrt separator. + (Currently supports strlist, strtuple, pathlist, pathtuple) + """ + sep = ',' + helpsep = 'comma' + if name.startswith('path'): + sep = os.pathsep + helpsep = 'pathsep' + + klass = None + if name.endswith('list'): + klass = list + elif name.endswith('tuple'): + klass = tuple + + return sep, klass, helpsep + def check_str_list_tuple(option, opt, value): """ check function for strlist and strtuple type assumes value is comma-separated list returns list or tuple of strings """ - split = value.split(',') - if option.type == 'strlist': - return split - elif option.type == 'strtuple': - return tuple(split) - else: - err = _("check_strlist_strtuple: unsupported type %s" % option.type) + sep, klass, _ = what_str_list_tuple(option.type) + split = value.split(sep) + + if klass is None: + err = _gettext("check_strlist_strtuple: unsupported type %s" % option.type) raise OptionValueError(err) + else: + return klass(split) class ExtOption(CompleterOption): @@ -97,7 +115,10 @@ class ExtOption(CompleterOption): - confighelp : hook for configfile-style help messages - store_debuglog : turns on fancylogger debugloglevel - also: 'store_infolog', 'store_warninglog' - - extend : extend default list (or create new one if is None) + - add : add value to default (result is default + value) + - add_first : add default to value (result is value + default) + - extend : alias for add with strlist type + - type must support + (__add__) and one of negate (__neg__) or slicing (__getslice__) - date : convert into datetime.date - datetime : convert into datetime.datetime - regex: compile str in regexp @@ -105,13 +126,18 @@ class ExtOption(CompleterOption): - set default to None if no option passed, - set to default if option without value passed, - set to value if option with value passed + + Types: + - strlist, strtuple : convert comma-separated string in a list resp. tuple of strings + - pathlist, pathtuple : using os.pathsep, convert pathsep-separated string in a list resp. tuple of strings + - the path separator is OS-dependent """ EXTEND_SEPARATOR = ',' ENABLE = 'enable' # do nothing DISABLE = 'disable' # inverse action - EXTOPTION_EXTRA_OPTIONS = ('extend', 'date', 'datetime', 'regex',) + EXTOPTION_EXTRA_OPTIONS = ('date', 'datetime', 'regex', 'add', 'add_first',) EXTOPTION_STORE_OR = ('store_or_None',) # callback type EXTOPTION_LOG = ('store_debuglog', 'store_infolog', 'store_warninglog',) EXTOPTION_HELP = ('shorthelp', 'confighelp',) @@ -121,10 +147,9 @@ class ExtOption(CompleterOption): TYPED_ACTIONS = Option.TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS + EXTOPTION_STORE_OR ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS - TYPE_CHECKER = dict([('strlist', check_str_list_tuple), - ('strtuple', check_str_list_tuple), - ] + Option.TYPE_CHECKER.items()) - TYPES = tuple(['strlist', 'strtuple'] + list(Option.TYPES)) + TYPE_STRLIST = ['%s%s' % (name, klass) for klass in ['list', 'tuple'] for name in ['str', 'path'] ] + TYPE_CHECKER = dict([(x, check_str_list_tuple) for x in TYPE_STRLIST] + Option.TYPE_CHECKER.items()) + TYPES = tuple(TYPE_STRLIST + list(Option.TYPES)) BOOLEAN_ACTIONS = ('store_true', 'store_false',) + EXTOPTION_LOG def __init__(self, *args, **kwargs): @@ -135,7 +160,11 @@ def __init__(self, *args, **kwargs): def _set_attrs(self, attrs): """overwrite _set_attrs to allow store_or callbacks""" Option._set_attrs(self, attrs) - if self.action in self.EXTOPTION_STORE_OR: + if self.action == 'extend': + # alias + self.action = 'add' + self.type = 'strlist' + elif self.action in self.EXTOPTION_STORE_OR: setattr(self, 'store_or', self.action) def store_or(option, opt_str, value, parser, *args, **kwargs): @@ -143,8 +172,7 @@ def store_or(option, opt_str, value, parser, *args, **kwargs): # see http://stackoverflow.com/questions/1229146/parsing-empty-options-in-python # ugly code, optparse is crap if parser.rargs and not parser.rargs[0].startswith('-'): - val = parser.rargs[0] - parser.rargs.pop(0) + val = option.check_value(opt_str, parser.rargs.pop(0)) else: val = kwargs.get('orig_default', None) @@ -157,13 +185,14 @@ def store_or(option, opt_str, value, parser, *args, **kwargs): self.type = 'string' self.callback = store_or - self.callback_kwargs = {'orig_default': copy.deepcopy(self.default), - } + self.callback_kwargs = { + 'orig_default': copy.deepcopy(self.default), + } self.action = 'callback' # act as callback if self.store_or == 'store_or_None': self.default = None else: - raise ValueError("_set_attrs: unknown store_or %s" % self.store_or) + self.log.raiseException("_set_attrs: unknown store_or %s" % self.store_or, exception=ValueError) def take_action(self, action, dest, opt, value, values, parser): """Extended take_action""" @@ -201,22 +230,29 @@ def take_action(self, action, dest, opt, value, values, parser): Option.take_action(self, action, dest, opt, value, values, parser) elif action in self.EXTOPTION_EXTRA_OPTIONS: - if action == "extend": - # comma separated list convert in list - lvalue = value.split(self.EXTEND_SEPARATOR) - values.ensure_value(dest, []).extend(lvalue) + if action in ("add", "add_first",): + # determine type from lvalue + # set default first + values.ensure_value(dest, type(value)()) + default = getattr(values, dest) + if not (hasattr(default, '__add__') and + (hasattr(default, '__neg__') or hasattr(default, '__getslice__'))): + msg = "Unsupported type %s for action %s (requires + and one of negate or slice)" + self.log.raiseException(msg % (type(default), action)) + if action == 'add': + lvalue = default + value + elif action == 'add_first': + lvalue = value + default elif action == "date": lvalue = date_parser(value) - setattr(values, dest, lvalue) elif action == "datetime": lvalue = datetime_parser(value) - setattr(values, dest, lvalue) elif action == "regex": lvalue = re.compile(r'' + value) - setattr(values, dest, lvalue) else: - raise(Exception("Unknown extended option action %s (known: %s)" % - (action, self.EXTOPTION_EXTRA_OPTIONS))) + msg = "Unknown extended option action %s (known: %s)" + self.log.raiseException(msg % (action, self.EXTOPTION_EXTRA_OPTIONS)) + setattr(values, dest, lvalue) else: Option.take_action(self, action, dest, opt, value, values, parser) @@ -291,7 +327,7 @@ def _process_short_opts(self, rargs, values): class ExtOptionGroup(OptionGroup): """An OptionGroup with support for configfile section names""" - RESERVED_SECTIONS = ['DEFAULT'] + RESERVED_SECTIONS = [ConfigParser.DEFAULTSECT] NO_SECTION = ('NO', 'SECTION') def __init__(self, *args, **kwargs): @@ -617,6 +653,9 @@ class GeneralOption(object): - go_useconfigfiles : use configfiles or not (default set by CONFIGFILES_USE) if True, an option --configfiles will be added - go_configfiles : list of configfiles to parse. Uses ConfigParser.read; last file wins + - go_configfiles_initenv : section dict of key/value dict; inserted before configfileparsing + As a special case, using all uppercase key in DEFAULT section with a case-sensitive + configparser can be used to set "constants" for easy interpolation in all sections. - go_loggername : name of logger, default classname - go_mainbeforedefault : set the main options before the default ones - go_autocompleter : dict with named options to pass to the autocomplete call (eg arg_completer) @@ -648,7 +687,8 @@ class GeneralOption(object): CONFIGFILES_INIT = [] # initial list of defaults, overwritten by go_configfiles options CONFIGFILES_IGNORE = [] CONFIGFILES_MAIN_SECTION = 'MAIN' # sectionname that contains the non-grouped/non-prefixed options - CONFIGFILE_PARSER = ConfigParser.ConfigParser + CONFIGFILE_PARSER = ConfigParser.SafeConfigParser + CONFIGFILE_CASESENSITIVE = True METAVAR_DEFAULT = True # generate a default metavar METAVAR_MAP = None # metvar, list of longopts map @@ -659,6 +699,7 @@ class GeneralOption(object): VERSION = None # set the version (will add --version) + DEFAULTSECT = ConfigParser.DEFAULTSECT DEFAULT_LOGLEVEL = None DEFAULT_CONFIGFILES = None DEFAULT_IGNORECONFIGFILES = None @@ -667,7 +708,8 @@ def __init__(self, **kwargs): go_args = kwargs.pop('go_args', None) self.no_system_exit = kwargs.pop('go_nosystemexit', None) # unit test option self.use_configfiles = kwargs.pop('go_useconfigfiles', self.CONFIGFILES_USE) # use or ignore config files - self.configfiles = kwargs.pop('go_configfiles', self.CONFIGFILES_INIT) # configfiles to parse + self.configfiles = kwargs.pop('go_configfiles', self.CONFIGFILES_INIT[:]) # configfiles to parse + configfiles_initenv = kwargs.pop('go_configfiles_initenv', None) # initial environment for configfiles to parse prefixloggername = kwargs.pop('go_prefixloggername', False) # name of logger is same as envvar prefix mainbeforedefault = kwargs.pop('go_mainbeforedefault', False) # Set the main options before the default ones autocompleter = kwargs.pop('go_autocompleter', {}) # Pass these options to the autocomplete call @@ -682,7 +724,7 @@ def __init__(self, **kwargs): self.parser = self.PARSER(**kwargs) self.parser.allow_interspersed_args = self.INTERSPERSED - self.configfile_parser = self.CONFIGFILE_PARSER() + self.configfile_parser = None self.configfile_remainder = {} loggername = self.__class__.__name__ @@ -717,6 +759,7 @@ def __init__(self, **kwargs): if not self.options is None: # None for eg usage/help + self.configfile_parser_init(initenv=configfiles_initenv) self.parseconfigfiles() self._set_default_loglevel() @@ -760,8 +803,8 @@ def _set_default_loglevel(self): def _make_configfiles_options(self): """Add configfiles option""" opts = { - 'configfiles': ("Parse (additional) configfiles", None, "extend", self.DEFAULT_CONFIGFILES), - 'ignoreconfigfiles': ("Ignore configfiles", None, "extend", self.DEFAULT_IGNORECONFIGFILES), + 'configfiles': ("Parse (additional) configfiles", "strlist", "add", self.DEFAULT_CONFIGFILES), + 'ignoreconfigfiles': ("Ignore configfiles", "strlist", "add", self.DEFAULT_IGNORECONFIGFILES), } descr = ['Configfile options', ''] self.log.debug("Add configfiles options descr %s opts %s (no prefix)" % (descr, opts)) @@ -880,16 +923,17 @@ def add_group_parser(self, opt_dict, description, prefix=None, otherdefaults=Non default = otherdefaults.get(key) extra_help = [] - if action in ("extend",) or typ in ('strlist', 'strtuple',): - extra_help.append("type comma-separated list") + if typ in ExtOption.TYPE_STRLIST: + sep, klass, helpsep = what_str_list_tuple(typ) + extra_help.append("type %s-separated %s" % (helpsep, klass.__name__)) elif typ is not None: extra_help.append("type %s" % typ) if default is not None: if len(str(default)) == 0: extra_help.append("def ''") # empty string - elif action in ("extend",) or typ in ('strlist', 'strtuple',): - extra_help.append("def %s" % ','.join(default)) + elif typ in ExtOption.TYPE_STRLIST: + extra_help.append("def %s" % sep.join(default)) else: extra_help.append("def %s" % default) @@ -983,14 +1027,13 @@ def parseoptions(self, options_list=None): try: (self.options, self.args) = self.parser.parse_args(options_list) except SystemExit, err: + try: + msg = err.message + except AttributeError: + # py2.4 + msg = str(err) + self.log.debug("parseoptions: parse_args err %s code %s" % (msg, err.code)) if self.no_system_exit: - try: - msg = err.message - except: - # py2.4 - msg = '_nomessage_' - self.log.debug("parseoptions: no_system_exit set after parse_args err %s code %s" % - (msg, err.code)) return else: sys.exit(err.code) @@ -1006,6 +1049,40 @@ def parseoptions(self, options_list=None): self.log.debug("Found options %s args %s" % (self.options, self.args)) + def configfile_parser_init(self, initenv=None): + """ + Initialise the confgiparser to use. + + @params initenv: insert initial environment into the configparser. + It is a dict of dicts; the first level key is the section name; + the 2nd level key,value is the key=value. + All section names, keys and values are converted to strings. + """ + self.configfile_parser = self.CONFIGFILE_PARSER() + + # make case sensitive + if self.CONFIGFILE_CASESENSITIVE: + self.log.debug('Initialise case sensitive configparser') + self.configfile_parser.optionxform = str + else: + self.log.debug('Initialise case insensitive configparser') + self.configfile_parser.optionxform = str.lower + + # insert the initenv in the parser + if initenv is None: + initenv = {} + + for name, section in initenv.items(): + name = str(name) + if name == self.DEFAULTSECT: + # is protected/reserved (and hidden) + pass + elif not self.configfile_parser.has_section(name): + self.configfile_parser.add_section(name) + + for key, value in section.items(): + self.configfile_parser.set(name, str(key), str(value)) + def parseconfigfiles(self): """Parse configfiles""" if not self.use_configfiles: @@ -1051,7 +1128,8 @@ def parseconfigfiles(self): self.log.debug("parseconfigfiles: following files were parsed %s" % parsed_files) self.log.debug("parseconfigfiles: following files were NOT parsed %s" % [x for x in configfiles if not x in parsed_files]) - self.log.debug("parseconfigfiles: sections (w/o DEFAULT) %s" % self.configfile_parser.sections()) + self.log.debug("parseconfigfiles: sections (w/o %s) %s" % + (self.DEFAULTSECT, self.configfile_parser.sections())) # walk through list of section names # - look for options set though config files @@ -1073,7 +1151,7 @@ def parseconfigfiles(self): if section not in cfg_sections_flat: self.log.debug("parseconfigfiles: found section %s, adding to remainder" % section) remainder = self.configfile_remainder.setdefault(section, {}) - # parse te remaining options, sections starting with 'raw_' + # parse the remaining options, sections starting with 'raw_' # as their name will be considered raw sections for opt, val in self.configfile_parser.items(section, raw=(section.startswith('raw_'))): remainder[opt] = val @@ -1098,8 +1176,16 @@ def parseconfigfiles(self): opt_name, opt_dest = self.make_options_option_name_and_destination(prefix, opt) actual_option = self.parser.get_option_by_long_name(opt_name) if actual_option is None: - self.log.raiseException('parseconfigfiles: no option corresponding with dest %s' % - opt_dest) + # don't fail on DEFAULT UPPERCASE options in case-sensitive mode. + in_def = self.configfile_parser.has_option(self.DEFAULTSECT, opt) + if in_def and self.CONFIGFILE_CASESENSITIVE and opt == opt.upper(): + self.log.debug(('parseconfigfiles: no option corresponding with ' + 'opt %s dest %s in section %s but found all uppercase ' + 'in DEFAULT section. Skipping.') % (opt, opt_dest, section)) + continue + else: + self.log.raiseException(('parseconfigfiles: no option corresponding with ' + 'opt %s dest %s in section %s') % (opt, opt_dest, section)) configfile_options_default[opt_dest] = actual_option.default @@ -1254,8 +1340,8 @@ def dict_by_prefix(self, merge_empty_prefix=False): return subdict def generate_cmd_line(self, ignore=None, add_default=None): - """Create the commandline options that would create the current self.options - opt_name is destination + """Create the commandline options that would create the current self.options. + The result is sorted on the destination names. @param ignore : regex on destination @param add_default : print value that are equal to default @@ -1309,7 +1395,11 @@ def generate_cmd_line(self, ignore=None, add_default=None): else: self.log.debug("generate_cmd_line %s adding %s non-default value %s" % (action, opt_name, opt_value)) - args.append("--%s=%s" % (opt_name, shell_quote(opt_value))) + if typ in ExtOption.TYPE_STRLIST: + sep, _, _ = what_str_list_tuple(typ) + args.append("--%s=%s" % (opt_name, shell_quote(sep.join(opt_value)))) + else: + args.append("--%s=%s" % (opt_name, shell_quote(opt_value))) elif action in ("store_true", "store_false",) + ExtOption.EXTOPTION_LOG: # not default! self.log.debug("generate_cmd_line adding %s value %s. store action found" % @@ -1334,23 +1424,39 @@ def generate_cmd_line(self, ignore=None, add_default=None): (opt_name, action, default)) else: args.append("--%s" % opt_name) - elif action in ("extend",): - # comma separated - self.log.debug("generate_cmd_line adding %s value %s. extend action, return as comma-separated list" % - (opt_name, opt_value)) - + elif action in ("add", "add_first"): if default is not None: - # remove these. if default is set, extend extends the default! - for def_el in default: - opt_value.remove(def_el) + if hasattr(opt_value, '__neg__'): + if action == 'add_first': + opt_value = opt_value + -default + else: + opt_value = -default + opt_value + elif hasattr(opt_value, '__getslice__'): + if action == 'add_first': + opt_value = opt_value[:-len(default)] + else: + opt_value = opt_value[len(default):] - if len(opt_value) == 0: - self.log.debug('generate_cmd_line skipping.') + if typ in ExtOption.TYPE_STRLIST: + sep, klass, helpsep = what_str_list_tuple(typ) + restype = '%s-separated %s' % (helpsep, klass.__name__) + value = sep.join(opt_value) + else: + restype = 'string' + value = opt_value + + if not opt_value: + # empty strings, empty lists, 0 + self.log.debug('generate_cmd_line no value left, skipping.') continue - args.append("--%s=%s" % (opt_name, shell_quote(",".join(opt_value)))) - elif typ in ('strlist', 'strtuple',): - args.append("--%s=%s" % (opt_name, shell_quote(",".join(opt_value)))) + self.log.debug("generate_cmd_line adding %s value %s. %s action, return as %s" % + (opt_name, opt_value, action, restype)) + + args.append("--%s=%s" % (opt_name, shell_quote(value))) + elif typ in ExtOption.TYPE_STRLIST: + sep, _, _ = what_str_list_tuple(typ) + args.append("--%s=%s" % (opt_name, shell_quote(sep.join(opt_value)))) elif action in ("append",): # add multiple times self.log.debug("generate_cmd_line adding %s value %s. append action, return as multiple args" % diff --git a/vsc/utils/missing.py b/vsc/utils/missing.py index 9d4768fec0..683269cb00 100644 --- a/vsc/utils/missing.py +++ b/vsc/utils/missing.py @@ -40,14 +40,20 @@ @author: Andy Georges (Ghent University) @author: Stijn De Weirdt (Ghent University) """ +import os +import re import shlex import subprocess +import sys import time from vsc.utils import fancylogger from vsc.utils.frozendict import FrozenDict +_log = fancylogger.getLogger('vsc.utils.missing') + + def partial(func, *args, **keywords): """ Return a new partial object which when called will behave like func called with the positional arguments args @@ -289,15 +295,105 @@ def shell_unquote(x): return shlex.split(str(x))[0] -def get_subclasses(klass): - """Get all subclasses recursively""" - res = [] - for cl in klass.__subclasses__(): - res.extend(get_subclasses(cl)) - res.append(cl) +def get_class_for(modulepath, class_name): + """ + Get class for a given Python class name and Python module path. + + @param modulepath: Python module path (e.g., 'vsc.utils.generaloption') + @param class_name: Python class name (e.g., 'GeneralOption') + """ + # try to import specified module path, reraise ImportError if it occurs + try: + module = __import__(modulepath, globals(), locals(), ['']) + except ImportError, err: + raise ImportError(err) + # try to import specified class name from specified module path, throw ImportError if this fails + try: + klass = getattr(module, class_name) + except AttributeError, err: + raise ImportError("Failed to import %s from %s: %s" % (class_name, modulepath, err)) + return klass + + +def get_subclasses_dict(klass, include_base_class=False): + """Get dict with subclasses per classes, recursively from the specified base class.""" + res = {} + subclasses = klass.__subclasses__() + if include_base_class: + res.update({klass: subclasses}) + for subclass in subclasses: + # always include base class for recursive call + res.update(get_subclasses_dict(subclass, include_base_class=True)) return res +def get_subclasses(klass, include_base_class=False): + """Get list of all subclasses, recursively from the specified base class.""" + return get_subclasses_dict(klass, include_base_class=include_base_class).keys() + + +def modules_in_pkg_path(pkg_path): + """Return list of module files in specified package path.""" + # if the specified (relative) package path doesn't exist, try and determine the absolute path via sys.path + if not os.path.isabs(pkg_path) and not os.path.isdir(pkg_path): + _log.debug("Obtained non-existing relative package path '%s', will try to figure out absolute path" % pkg_path) + newpath = None + for sys_path_dir in sys.path: + abspath = os.path.join(sys_path_dir, pkg_path) + if os.path.isdir(abspath): + _log.debug("Found absolute path %s for package path %s, verifying it" % (abspath, pkg_path)) + # also make sure an __init__.py is in place in every subdirectory + is_pkg = True + subdir = '' + for pkg_path_dir in pkg_path.split(os.path.sep): + subdir = os.path.join(subdir, pkg_path_dir) + if not os.path.isfile(os.path.join(sys_path_dir, subdir, '__init__.py')): + is_pkg = False + tup = (subdir, abspath, pkg_path) + _log.debug("No __init__.py found in %s, %s is not a valid absolute path for pkg_path %s" % tup) + break + if is_pkg: + newpath = abspath + break + + if newpath is None: + # give up if we couldn't find an absolute path for the imported package + tup = (pkg_path, sys.path) + raise OSError("Can't browse package via non-existing relative path '%s', not found in sys.path (%s)" % tup) + else: + pkg_path = newpath + _log.debug("Found absolute package path %s" % pkg_path) + + module_regexp = re.compile(r"^(?P[^_].*|__init__)\.py$") + modules = [res.group('modname') for res in map(module_regexp.match, os.listdir(pkg_path)) if res] + _log.debug("List of modules for package in %s: %s" % (pkg_path, modules)) + return modules + + +def avail_subclasses_in(base_class, pkg_name, include_base_class=False): + """Determine subclasses for specificied base classes in modules in (only) specified packages.""" + + def try_import(name): + """Try import the specified package/module.""" + try: + # don't use return value of __import__ since it may not be the package itself but it's parent + __import__(name, globals()) + return sys.modules[name] + except ImportError, err: + raise ImportError("avail_subclasses_in: failed to import %s: %s" % (name, err)) + + # import all modules in package path(s) before determining subclasses + pkg = try_import(pkg_name) + for pkg_path in pkg.__path__: + for mod in modules_in_pkg_path(pkg_path): + # no need to directly import __init__ (already done by importing package) + if not mod.startswith('__init__'): + _log.debug("Importing module '%s' from package '%s'" % (mod, pkg_name)) + try_import('%s.%s' % (pkg_name, mod)) + + return get_subclasses_dict(base_class, include_base_class=include_base_class) + + class TryOrFail(object): """ Perform the function n times, catching each exception in the exception tuple except on the last try @@ -310,16 +406,15 @@ def __init__(self, n, exceptions=(Exception,), sleep=0): def __call__(self, function): def new_function(*args, **kwargs): - log = fancylogger.getLogger(function.__name__) for i in xrange(0, self.n): try: return function(*args, **kwargs) except self.exceptions, err: if i == self.n - 1: raise - log.exception("try_or_fail caught an exception - attempt %d: %s" % (i, err)) + _log.exception("try_or_fail caught an exception - attempt %d: %s" % (i, err)) if self.sleep > 0: - log.warning("try_or_fail is sleeping for %d seconds before the next attempt" % (self.sleep,)) + _log.warning("try_or_fail is sleeping for %d seconds before the next attempt" % (self.sleep,)) time.sleep(self.sleep) return new_function diff --git a/vsc/utils/rest.py b/vsc/utils/rest.py index 82d5835eee..18974a8167 100644 --- a/vsc/utils/rest.py +++ b/vsc/utils/rest.py @@ -42,7 +42,11 @@ import simplejson as json from vsc.utils import fancylogger -from vsc.utils.missing import partial + +try: + from functools import partial +except ImportError: + from vsc.utils.missing import partial class Client(object): diff --git a/vsc/utils/testing.py b/vsc/utils/testing.py new file mode 100644 index 0000000000..71807d5677 --- /dev/null +++ b/vsc/utils/testing.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +## +# +# Copyright 2014-2014 Ghent University +# +# This file is part of vsc-base, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/vsc-base +# +# vsc-base is free software: you can redistribute it and/or modify +# it under the terms of the GNU Library General Public License as +# published by the Free Software Foundation, either version 2 of +# the License, or (at your option) any later version. +# +# vsc-base is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU Library General Public License +# along with vsc-base. If not, see . +## +""" +Test utilities. + +@author: Kenneth Hoste (Ghent University) +""" + +import re +import sys +from cStringIO import StringIO +from unittest import TestCase + + +class EnhancedTestCase(TestCase): + """Enhanced test case, provides extra functionality (e.g. an assertErrorRegex method).""" + + def setUp(self): + """Prepare test case.""" + super(EnhancedTestCase, self).setUp() + self.orig_sys_stdout = sys.stdout + self.orig_sys_stderr = sys.stderr + + def convert_exception_to_str(self, err): + """Convert an Exception instance to a string.""" + msg = err + if hasattr(err, 'msg'): + msg = err.msg + elif hasattr(err, 'message'): + msg = err.message + if not msg: + # rely on str(msg) in case err.message is empty + msg = err + elif hasattr(err, 'args'): # KeyError in Python 2.4 only provides message via 'args' attribute + msg = err.args[0] + else: + msg = err + try: + res = str(msg) + except UnicodeEncodeError: + res = msg.encode('utf8', 'replace') + + return res + + def assertErrorRegex(self, error, regex, call, *args, **kwargs): + """ + Convenience method to match regex with the expected error message. + Example: self.assertErrorRegex(OSError, "No such file or directory", os.remove, '/no/such/file') + """ + try: + call(*args, **kwargs) + str_kwargs = ['='.join([k, str(v)]) for (k, v) in kwargs.items()] + str_args = ', '.join(map(str, args) + str_kwargs) + self.assertTrue(False, "Expected errors with %s(%s) call should occur" % (call.__name__, str_args)) + except error, err: + msg = self.convert_exception_to_str(err) + if isinstance(regex, basestring): + regex = re.compile(regex) + self.assertTrue(regex.search(msg), "Pattern '%s' is found in '%s'" % (regex.pattern, msg)) + + def mock_stdout(self, enable): + """Enable/disable mocking stdout.""" + sys.stdout.flush() + if enable: + sys.stdout = StringIO() + else: + sys.stdout = self.orig_sys_stdout + + def mock_stderr(self, enable): + """Enable/disable mocking stdout.""" + sys.stderr.flush() + if enable: + sys.stderr = StringIO() + else: + sys.stderr = self.orig_sys_stderr + + def get_stdout(self): + """Return output captured from stdout until now.""" + return sys.stdout.getvalue() + + def get_stderr(self): + """Return output captured from stderr until now.""" + return sys.stderr.getvalue() + + def tearDown(self): + """Cleanup after running a test.""" + self.mock_stdout(False) + self.mock_stderr(False) + super(EnhancedTestCase, self).tearDown()