add support for implementing pre- and post-step hooks

easybuild/framework/easyblock.py

@@ -71,8 +71,10 @@
 from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, convert_name
 from easybuild.tools.filetools import compute_checksum, copy_file, derive_alt_pypi_url, diff_files, download_file
 from easybuild.tools.filetools import encode_class_name, extract_file, is_alt_pypi_url, mkdir, move_logs, read_file
-from easybuild.tools.filetools import remove_file, rmtree2, write_file
-from easybuild.tools.filetools import verify_checksum, weld_paths
+from easybuild.tools.filetools import remove_file, rmtree2, verify_checksum, weld_paths, write_file
+from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
+from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTPROC_STEP, PREPARE_STEP
+from easybuild.tools.hooks import READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP, run_hook
 from easybuild.tools.run import run_cmd
 from easybuild.tools.jenkins import write_to_xml
 from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for
@@ -88,23 +90,6 @@
 from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION
 
 
-BUILD_STEP = 'build'
-CLEANUP_STEP = 'cleanup'
-CONFIGURE_STEP = 'configure'
-EXTENSIONS_STEP = 'extensions'
-FETCH_STEP = 'fetch'
-MODULE_STEP = 'module'
-PACKAGE_STEP = 'package'
-PATCH_STEP = 'patch'
-PERMISSIONS_STEP = 'permissions'
-POSTPROC_STEP = 'postproc'
-PREPARE_STEP = 'prepare'
-READY_STEP = 'ready'
-SANITYCHECK_STEP = 'sanitycheck'
-SOURCE_STEP = 'source'
-TEST_STEP = 'test'
-TESTCASES_STEP = 'testcases'
-
 MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP]
 
 # string part of URL for Python packages on PyPI that indicates needs to be rewritten (see derive_alt_pypi_url)
@@ -136,7 +121,7 @@ def extra_options(extra=None):
     #
     # INIT
     #
-    def __init__(self, ec):
+    def __init__(self, ec, hooks=None):
         """
         Initialize the EasyBlock instance.
         :param ec: a parsed easyconfig file (EasyConfig instance)
@@ -145,6 +130,9 @@ def __init__(self, ec):
         # keep track of original working directory, so we can go back there
         self.orig_workdir = os.getcwd()
 
+        # list of pre- and post-step hooks
+        self.hooks = hooks or []
+
         # list of patch/source files, along with checksums
         self.patches = []
         self.src = []
@@ -2434,6 +2422,9 @@ def run_step(self, step, step_methods):
         """
         self.log.info("Starting %s step", step)
         self.update_config_template_run_step()
+
+        run_hook(step, self.hooks, pre_step_hook=True, args=[self])
+
         for step_method in step_methods:
             self.log.info("Running method %s part of step %s" % ('_'.join(step_method.func_code.co_names), step))
 
@@ -2457,6 +2448,8 @@ def run_step(self, step, step_methods):
                 # and returns the actual method, so use () to execute it
                 step_method(self)()
 
+        run_hook(step, self.hooks, post_step_hook=True, args=[self])
+
         if self.cfg['stop'] == step:
             self.log.info("Stopping after %s step.", step)
             raise StopException(step)
@@ -2499,7 +2492,7 @@ def prepare_step_spec(initial):
             (False, lambda x: x.make_installdir),
             (True, lambda x: x.install_step),
         ]
-        install_step_spec = lambda initial: get_step('install', "installing", install_substeps, True, initial=initial)
+        install_step_spec = lambda init: get_step(INSTALL_STEP, "installing", install_substeps, True, initial=init)
 
         # format for step specifications: (stop_name: (description, list of functions, skippable))
 
@@ -2601,7 +2594,7 @@ def print_dry_run_note(loc, silent=True):
     dry_run_msg(msg, silent=silent)
 
 
-def build_and_install_one(ecdict, init_env):
+def build_and_install_one(ecdict, init_env, hooks=None):
     """
     Build the software
     :param ecdict: dictionary contaning parsed easyconfig + metadata
@@ -2639,7 +2632,7 @@ def build_and_install_one(ecdict, init_env):
     try:
         app_class = get_easyblock_class(easyblock, name=name)
 
-        app = app_class(ecdict['ec'])
+        app = app_class(ecdict['ec'], hooks=hooks)
         _log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock))
     except EasyBuildError, err:
         print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg),

easybuild/main.py

@@ -61,6 +61,7 @@
 from easybuild.tools.filetools import adjust_permissions, cleanup, write_file
 from easybuild.tools.github import check_github, find_easybuild_easyconfig, install_github_token
 from easybuild.tools.github import new_pr, merge_pr, update_pr
+from easybuild.tools.hooks import START, END, load_hooks, run_hook
 from easybuild.tools.modules import modules_tool
 from easybuild.tools.options import parse_external_modules_metadata, process_software_build_specs, use_color
 from easybuild.tools.robot import check_conflicts, det_robot_path, dry_run, resolve_dependencies, search_easyconfigs
@@ -104,18 +105,27 @@ def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=
     return [(ec_file, generated)]
 
 
-def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
-    """Build and install software for all provided parsed easyconfig files."""
+def build_and_install_software(ecs, init_session_state, exit_on_failure=True, hooks=None):
+    """
+    Build and install software for all provided parsed easyconfig files.
+
+    :param ecs: easyconfig files to install software with
+    :param init_session_state: initial session state, to use in test reports
+    :param exit_on_failure: whether or not to exit on installation failure
+    :param hooks: list of defined pre- and post-step hooks
+    """
     # obtain a copy of the starting environment so each build can start afresh
     # we shouldn't use the environment from init_session_state, since relevant env vars might have been set since
     # e.g. via easyconfig.handle_allowed_system_deps
     init_env = copy.deepcopy(os.environ)
 
+    run_hook(START, hooks)
+
     res = []
     for ec in ecs:
         ec_res = {}
         try:
-            (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env)
+            (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, hooks=hooks)
             ec_res['log_file'] = app_log
             if not ec_res['success']:
                 ec_res['err'] = EasyBuildError(err)
@@ -154,6 +164,8 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
 
         res.append((ec, ec_res))
 
+    run_hook(END, hooks)
+
     return res
 
 
@@ -453,9 +465,12 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
             sys.exit(0)
 
     # build software, will exit when errors occurs (except when testing)
-    exit_on_failure = not options.dump_test_report and not options.upload_test_report
     if not testing or (testing and do_build):
-        ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
+        exit_on_failure = not (options.dump_test_report or options.upload_test_report)
+        hooks = load_hooks(options.hooks)
+
+        ecs_with_res = build_and_install_software(ordered_ecs, init_session_state,
+                                                  exit_on_failure=exit_on_failure, hooks=hooks)
     else:
         ecs_with_res = [(ec, {}) for ec in ordered_ecs]
 

easybuild/tools/config.py

@@ -121,10 +121,6 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'force_download',
         'from_pr',
         'git_working_dirs_path',
-        'pr_branch_name',
-        'pr_target_account',
-        'pr_target_branch',
-        'pr_target_repo',
         'github_user',
         'github_org',
         'group',
@@ -142,6 +138,10 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'optarch',
         'package_tool_options',
         'parallel',
+        'pr_branch_name',
+        'pr_target_account',
+        'pr_target_branch',
+        'pr_target_repo',
         'rpath_filter',
         'regtest_output_dir',
         'skip',

easybuild/tools/filetools.py

@@ -1,4 +1,4 @@
-
+# #
 # Copyright 2009-2017 Ghent University
 #
 # This file is part of EasyBuild,

easybuild/tools/hooks.py

@@ -0,0 +1,176 @@
+# #
+# Copyright 2017-2017 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+# #
+"""
+Hook support.
+
+:author: Kenneth Hoste (Ghent University)
+"""
+import difflib
+import imp
+import os
+from vsc.utils import fancylogger
+
+from easybuild.tools.build_log import EasyBuildError
+
+
+_log = fancylogger.getLogger('hooks', fname=False)
+
+BUILD_STEP = 'build'
+CLEANUP_STEP = 'cleanup'
+CONFIGURE_STEP = 'configure'
+EXTENSIONS_STEP = 'extensions'
+FETCH_STEP = 'fetch'
+INSTALL_STEP = 'install'
+MODULE_STEP = 'module'
+PACKAGE_STEP = 'package'
+PATCH_STEP = 'patch'
+PERMISSIONS_STEP = 'permissions'
+POSTPROC_STEP = 'postproc'
+PREPARE_STEP = 'prepare'
+READY_STEP = 'ready'
+SANITYCHECK_STEP = 'sanitycheck'
+SOURCE_STEP = 'source'
+TEST_STEP = 'test'
+TESTCASES_STEP = 'testcases'
+
+START = 'start'
+END = 'end'
+
+PRE_PREF = 'pre_'
+POST_PREF = 'post_'
+HOOK_SUFF = '_hook'
+
+# list of names for steps in installation procedure (in order of execution)
+STEP_NAMES = [FETCH_STEP, READY_STEP, SOURCE_STEP, PATCH_STEP, PREPARE_STEP, CONFIGURE_STEP, BUILD_STEP, TEST_STEP,
+              INSTALL_STEP, EXTENSIONS_STEP, POSTPROC_STEP, SANITYCHECK_STEP, CLEANUP_STEP, MODULE_STEP,
+              PERMISSIONS_STEP, PACKAGE_STEP, TESTCASES_STEP]
+
+KNOWN_HOOKS = [h + HOOK_SUFF for h in [START] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END]]
+
+
+def load_hooks(hooks_path):
+    """Load defined hooks (if any)."""
+    hooks = {}
+
+    if hooks_path:
+        if not os.path.exists(hooks_path):
+            raise EasyBuildError("Specified path for hooks implementation does not exist: %s", hooks_path)
+
+        (hooks_filename, hooks_file_ext) = os.path.splitext(os.path.split(hooks_path)[1])
+        if hooks_file_ext == '.py':
+            _log.info("Importing hooks implementation from %s...", hooks_path)
+            try:
+                # import module that defines hooks, and collect all functions of which name ends with '_hook'
+                imported_hooks = imp.load_source(hooks_filename, hooks_path)
+                for attr in dir(imported_hooks):
+                    if attr.endswith(HOOK_SUFF):
+                        hook = getattr(imported_hooks, attr)
+                        if callable(hook):
+                            hooks.update({attr: hook})
+                        else:
+                            _log.debug("Skipping non-callable attribute '%s' when loading hooks", attr)
+                _log.info("Found hooks: %s", sorted(hooks.keys()))
+            except ImportError as err:
+                raise EasyBuildError("Failed to import hooks implementation from %s: %s", hooks_path, err)
+        else:
+            raise EasyBuildError("Provided path for hooks implementation should be location of a Python file (*.py)")
+    else:
+        _log.info("No location for hooks implementation provided, no hooks defined")
+
+    verify_hooks(hooks)
+
+    return hooks
+
+
+def verify_hooks(hooks):
+    """Check whether list of obtained hooks only includes known hooks."""
+    unknown_hooks = []
+    for key in hooks:
+        if key not in KNOWN_HOOKS:
+            unknown_hooks.append(key)
+
+    if unknown_hooks:
+        error_lines = ["Found one or more unknown hooks:"]
+
+        for unknown_hook in unknown_hooks:
+            error_lines.append("* %s" % unknown_hook)
+            # try to find close match, may be just a typo in the hook name
+            close_matching_hooks = difflib.get_close_matches(unknown_hook, KNOWN_HOOKS, 2, 0.8)
+            if close_matching_hooks:
+                error_lines[-1] += " (did you mean %s?)" % ', or '.join("'%s'" % h for h in close_matching_hooks)
+
+        error_lines.extend(['', "Run 'eb --avail-hooks' to get an overview of known hooks"])
+
+        raise EasyBuildError('\n'.join(error_lines))
+    else:
+        _log.info("Defined hooks verified, all known hooks: %s", ', '.join(h for h in hooks))
+
+
+def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False):
+    """
+    Find hook with specified label.
+
+    :param label: name of hook
+    :param hooks: list of defined hooks
+    :param pre_step_hook: indicates whether hook to run is a pre-step hook
+    :param post_step_hook: indicates whether hook to run is a post-step hook
+    """
+    res = None
+
+    if pre_step_hook:
+        hook_prefix = PRE_PREF
+    elif post_step_hook:
+        hook_prefix = POST_PREF
+    else:
+        hook_prefix = ''
+
+    hook_name = hook_prefix + label + HOOK_SUFF
+
+    for key in hooks:
+        if key == hook_name:
+            _log.info("Found %s hook", hook_name)
+            res = hooks[key]
+            break
+
+    return res
+
+
+def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None):
+    """
+    Run hook with specified label.
+
+    :param label: name of hook
+    :param hooks: list of defined hooks
+    :param pre_step_hook: indicates whether hook to run is a pre-step hook
+    :param post_step_hook: indicates whether hook to run is a post-step hook
+    :param args: arguments to pass to hook function
+    """
+    hook = find_hook(label, hooks, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook)
+    if hook:
+        if args is None:
+            args = []
+
+        _log.info("Running %s hook (arguments: %s)...", hook.__name__, args)
+        hook(*args)