Merge pull request #1681 from hpcugent/develop

release EasyBuild v2.7.0

.gitignore

@@ -2,6 +2,7 @@
 .project
 LICENSE_HEADER
 *.pyc
+*.pyo
 *.nja
 build/
 dist/

MANIFEST.in

@@ -1,6 +1,7 @@
 include eb
 include minimal_bash_completion.bash
 include optcomplete.bash
+recursive-include etc *
 recursive-include easybuild *py
 recursive-include test *py *eb
 recursive-include test/framework/modules *

RELEASE_NOTES

@@ -3,6 +3,71 @@ For more detailed information, please see the git log.
 
 These release notes can also be consulted at http://easybuild.readthedocs.org/en/latest/Release_notes.html.
 
+v2.7.0 (March 20th 2016)
+------------------------
+
+feature + bugfix release
+- stabilize Cray support
+    - enable 'dynamic' toolchain option by default for Cray* toolchains (#1581)
+    - remove FFTW from the Cray toolchains definition (#1585)
+    - add external modules metadata for Cray systems (#1638)
+    - fix independency of Cray toolchains w.r.t. toolchain build environment (#1641, #1647)
+    - remove requirement to use --experimental for Cray toolchains (#1663)
+- enable Python optimization mode in 'eb' (#1357)
+- improved GitHub integration
+    - improve error handling on git commands + better logging for --new-pr/--update-pr (#1590)
+    - use git rather than https in --new-pr/--update-pr (#1602)
+    - add -u as shorthand for --upload-test-report (#1605)
+    - fix --from-pr for PRs that include renamed/deleted files (#1615)
+    - add support for --install-github-token and --check-github (#1616)
+    - fix fetch_easyconfigs_from_pr w.r.t. duplicate files in PRs (#1628)
+- various other enhancements, including:
+    - add support for --search-filename and --terse (#1577)
+    - support complete bash completion (#1580)
+    - add support for %(*ver)s and %(*shortver)s templates (#1595, #1604)
+        - incl. %(javaver)s, %(javashortver)s, %(perlver)s, %(perlshortver)s, %(pyver)s, %(pyshortver)s, %(rver)s, %(rshortver)s
+    - define HOME constant that can be used in easyconfig files (#1607)
+    - implement support for generating 'swap' statements in module files (#1609)
+    - add support for --show-config (#1611, #1620)
+    - simplified support for --minimal-toolchains (#1614, #1619, #1622, #1625, #1646)
+    - add support for --dump-env-script (#1624)
+    - enhance ModulesTool.exist to also recognize partial module names (#1630)
+    - improve error message for toolchain definition errors (#1631)
+    - make default is_short_modname_for check less strict to support versionless external modules as deps (#1632)
+    - mention hostname in comment made by --upload-test-report (#1635)
+    - support providing additional relative path for prefix in external module metadata (#1637)
+    - add ThematicModuleNamingScheme (#1645)
+    - enhance logging format: remove logger name, mention location instead (#1649, #1654)
+    - update kernel versions for SLES12 (#1659)
+    - raise EasyBuildError rather than ImportError in only_if_module_is_available decorator (#1662)
+- various bug fixes, including:
+    - fix Lmod spider output in generated modules (#1583)
+    - correctly define 'easybuild' namespaces (#1593, #1666, #1680)
+        - this change requires that the setuptools Python package is available (at runtime)
+        - using custom easyblocks by adding them in the Python search path ($PYTHONPATH) may require adjustments,
+          i.e. also using pkg_resources.declare_namespace in the __init__.py files;
+          we highly recommend to use --include-easyblocks instead,
+          see http://easybuild.readthedocs.org/en/latest/Including_additional_Python_modules.html
+        - note: this has the side-effect of not being able anymore to reliably use 'eb' in the parent directory of
+          the easybuild-framework repository (#1667)
+    - fix template for savannah.gnu.org source URL (#1601)
+    - stop running 'module purge', only restore environment (#1608)
+    - fix license headers: Hercules foundation is now FWO (#1629)
+    - avoid that fancylogger tries to import mpi4py to determine MPI rank (#1648)
+    - fix error in tests when 'file' backend is not available in Python keyring (#1650)
+    - update develop install script (#1651)
+    - handle allowed system deps during prepare_step rather than during parsing of easyconfig (#1652)
+    - add function to find FlexLM licenses: find_flexlm_license (#1633, #1653)
+    - fix availability check for external modules with partial module name (#1634, #1643)
+    - fix bootstrap script to ensure setuptools is also installed (#1655)
+    - fix issue in bootstrap script with vsc-base being picked up from the OS (#1656)
+    - fix bootstrap script for environment where 'python' is Python 3.x (#1660)
+    - remove --experimental for tests related to --package (#1665)
+    - ensure path to setuptools is included in $PYTHONPATH being used to test scripts (#1671)
+    - sanitize environment before initializing easyblocks (#1676)
+    - remove reload statements in include.py, since they are not required and break --include-toolchains (#1679)
+
+
 v2.6.0 (January 26th 2016)
 --------------------------
 

easybuild/__init__.py

@@ -1,11 +1,11 @@
 ##
-# Copyright 2011-2015 Ghent University
+# Copyright 2011-2016 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
 # with support of Ghent University (http://ugent.be/hpc),
 # the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en),
-# the Hercules foundation (http://www.herculesstichting.be/in_English)
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
 # and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
 #
 # http://github.com/hpcugent/easybuild
@@ -26,8 +26,16 @@
 Declares EasyBuild namespace, in an extendable way.
 
 @author: Jens Timmerman (Ghent University)
+@author: Kenneth Hoste (Ghent University)
 """
-from pkgutil import extend_path
+import os
+import pkg_resources
+import sys
 
-# we're not the only ones in this namespace
-__path__ = extend_path(__path__, __name__)  #@ReservedAssignment
+# check whether EasyBuild is being run from a directory that contains easybuild/__init__.py;
+# that doesn't work (fails with import errors), due to weirdness to Python packaging/setuptools/namespaces
+if __path__[0] == 'easybuild':
+    sys.stderr.write("ERROR: Running EasyBuild from %s does not work (Python packaging weirdness)...\n" % os.getcwd())
+    sys.exit(1)
+
+pkg_resources.declare_namespace(__name__)

easybuild/framework/__init__.py

@@ -1,11 +1,11 @@
 ##
-# Copyright 2009-2015 Ghent University
+# Copyright 2009-2016 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
 # with support of Ghent University (http://ugent.be/hpc),
 # the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en),
-# the Hercules foundation (http://www.herculesstichting.be/in_English)
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
 # and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
 #
 # http://github.com/hpcugent/easybuild

easybuild/framework/easyblock.py

@@ -1,11 +1,11 @@
 # #
-# Copyright 2009-2015 Ghent University
+# Copyright 2009-2016 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
 # with support of Ghent University (http://ugent.be/hpc),
 # the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en),
-# the Hercules foundation (http://www.herculesstichting.be/in_English)
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
 # and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
 #
 # http://github.com/hpcugent/easybuild
@@ -72,6 +72,7 @@
 from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator
 from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
 from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX
+from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name
 from easybuild.tools.modules import get_software_root, modules_tool
 from easybuild.tools.package.utilities import package
 from easybuild.tools.repository.repository import init_repository
@@ -1364,7 +1365,7 @@ def check_readiness_step(self):
         # - if a current module can be found, skip is ok
         # -- this is potentially very dangerous
         if self.cfg['skip']:
-            if self.modules_tool.exist([self.full_mod_name])[0]:
+            if self.modules_tool.exist([self.full_mod_name], skip_avail=True)[0]:
                 self.skip = True
                 self.log.info("Module %s found." % self.full_mod_name)
                 self.log.info("Going to skip actual main build and potential existing extensions. Expert only.")
@@ -1510,9 +1511,11 @@ def patch_step(self, beginpath=None):
             if not apply_patch(patch['path'], src, copy=copy_patch, level=level):
                 raise EasyBuildError("Applying patch %s failed", patch['name'])
 
-    def prepare_step(self):
+    def prepare_step(self, start_dir=True):
         """
         Pre-configure step. Set's up the builddir just before starting configure
+
+        @param start_dir: guess start directory based on unpacked sources
         """
         if self.dry_run:
             self.dry_run_msg("Defining build environment, based on toolchain (options) and specified dependencies...\n")
@@ -1523,8 +1526,16 @@ def prepare_step(self):
         # prepare toolchain: load toolchain module and dependencies, set up build environment
         self.toolchain.prepare(self.cfg['onlytcmod'], silent=self.silent)
 
+        # handle allowed system dependencies
+        for (name, version) in self.cfg['allow_system_deps']:
+            # root is set to name, not an actual path
+            env.setvar(get_software_root_env_var_name(name), name)
+            # version is expected to be something that makes sense
+            env.setvar(get_software_version_env_var_name(name), version)
+
         # guess directory to start configure/build/install process in, and move there
-        self.guess_start_dir()
+        if start_dir:
+            self.guess_start_dir()
 
     def configure_step(self):
         """Configure build  (abstract method)."""
@@ -2296,6 +2307,8 @@ def build_and_install_one(ecdict, init_env):
     # successful (non-dry-run) build
     if result and not dry_run:
 
+        ec_filename = '%s-%s.eb' % (app.name, det_full_ec_version(app.cfg))
+
         if app.cfg['stop']:
             ended = 'STOPPED'
             if app.builddir is not None:
@@ -2314,14 +2327,28 @@ def build_and_install_one(ecdict, init_env):
             buildstats = get_build_stats(app, start_time, build_option('command_line'))
             _log.info("Build stats: %s" % buildstats)
 
+            if build_option("minimal_toolchains"):
+                # for reproducability we dump out the parsed easyconfig since the contents are affected when
+                # --minimal-toolchains (and --use-existing-modules) is used
+                _log.debug("Dumping parsed easyconfig rather than original easyconfig to install dir")
+
+                # add the parsed file to the reproducability directory
+                # TODO --try-toolchain needs to be fixed so this doesn't play havoc with it's usability
+                repo_spec = os.path.join(new_log_dir, 'reprod', ec_filename)
+                app.cfg.dump(repo_spec)
+
+            else:
+                _log.debug("Dumping original easyconfig to install dir")
+                repo_spec = spec
+
             try:
                 # upload spec to central repository
                 currentbuildstats = app.cfg['buildstats']
                 repo = init_repository(get_repository(), get_repositorypath())
                 if 'original_spec' in ecdict:
                     block = det_full_ec_version(app.cfg) + ".block"
                     repo.add_easyconfig(ecdict['original_spec'], app.name, block, buildstats, currentbuildstats)
-                repo.add_easyconfig(spec, app.name, det_full_ec_version(app.cfg), buildstats, currentbuildstats)
+                repo.add_easyconfig(repo_spec, app.name, det_full_ec_version(app.cfg), buildstats, currentbuildstats)
                 repo.commit("Built %s" % app.full_mod_name)
                 del repo
             except EasyBuildError, err:
@@ -2333,7 +2360,7 @@ def build_and_install_one(ecdict, init_env):
         move_logs(app.logfile, application_log)
 
         try:
-            newspec = os.path.join(new_log_dir, "%s-%s.eb" % (app.name, det_full_ec_version(app.cfg)))
+            newspec = os.path.join(new_log_dir, ec_filename)
             # only copy if the files are not the same file already (yes, it happens)
             if os.path.exists(newspec) and os.path.samefile(spec, newspec):
                 _log.debug("Not copying easyconfig file %s to %s since files are identical" % (spec, newspec))
@@ -2418,6 +2445,9 @@ def build_easyconfigs(easyconfigs, output_dir, test_results):
     build_stopped = {}
     apploginfo = lambda x, y: x.log.info(y)
 
+    # sanitize environment before initialising easyblocks
+    sanitize_env()
+
     def perform_step(step, obj, method, logfile):
         """Perform method on object if it can be built."""
         if (isinstance(obj, dict) and obj['spec'] not in build_stopped) or obj not in build_stopped:

easybuild/framework/easyconfig/__init__.py

@@ -1,11 +1,11 @@
 # #
-# Copyright 2009-2015 Ghent University
+# Copyright 2009-2016 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
 # with support of Ghent University (http://ugent.be/hpc),
 # the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en),
-# the Hercules foundation (http://www.herculesstichting.be/in_English)
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
 # and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
 #
 # http://github.com/hpcugent/easybuild

easybuild/framework/easyconfig/constants.py

@@ -1,11 +1,11 @@
 #
-# Copyright 2013-2015 Ghent University
+# Copyright 2013-2016 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
 # with support of Ghent University (http://ugent.be/hpc),
 # the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en),
-# the Hercules foundation (http://www.herculesstichting.be/in_English)
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
 # and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
 #
 # http://github.com/hpcugent/easybuild
@@ -30,6 +30,7 @@
 @author: Stijn De Weirdt (Ghent University)
 @author: Kenneth Hoste (Ghent University)
 """
+import os
 import platform
 from vsc.utils import fancylogger
 
@@ -43,10 +44,11 @@
 # constants that can be used in easyconfig
 EASYCONFIG_CONSTANTS = {
     'EXTERNAL_MODULE': (EXTERNAL_MODULE_MARKER, "External module marker"),
-    'SYS_PYTHON_VERSION': (platform.python_version(), "System Python version (platform.python_version())"),
+    'HOME': (os.path.expanduser('~'), "Home directory ($HOME)"),
     'OS_TYPE': (get_os_type(), "System type (e.g. 'Linux' or 'Darwin')"),
     'OS_NAME': (get_os_name(), "System name (e.g. 'fedora' or 'RHEL')"),
     'OS_VERSION': (get_os_version(), "System version"),
+    'SYS_PYTHON_VERSION': (platform.python_version(), "System Python version (platform.python_version())"),
 }
 
 

easybuild/framework/easyconfig/default.py

@@ -1,11 +1,11 @@
 # #
-# Copyright 2009-2015 Ghent University
+# Copyright 2009-2016 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
 # with support of Ghent University (http://ugent.be/hpc),
 # the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en),
-# the Hercules foundation (http://www.herculesstichting.be/in_English)
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
 # and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
 #
 # http://github.com/hpcugent/easybuild