add support for implementing pre- and post-step hooks

easybuild/framework/easyblock.py

@@ -71,7 +71,8 @@
 from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, convert_name
 from easybuild.tools.filetools import compute_checksum, copy_file, derive_alt_pypi_url, diff_files, download_file
 from easybuild.tools.filetools import encode_class_name, extract_file, is_alt_pypi_url, mkdir, move_logs, read_file
-from easybuild.tools.filetools import remove_file, rmtree2, run_hook, write_file, verify_checksum, weld_paths
+from easybuild.tools.filetools import remove_file, rmtree2, verify_checksum, weld_paths, write_file
+from easybuild.tools.hooks import run_hook
 from easybuild.tools.run import run_cmd
 from easybuild.tools.jenkins import write_to_xml
 from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for

easybuild/main.py

@@ -58,9 +58,10 @@
 from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak
 from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option
 from easybuild.tools.docs import list_software
-from easybuild.tools.filetools import adjust_permissions, cleanup, load_hooks, run_hook, write_file
+from easybuild.tools.filetools import adjust_permissions, cleanup, write_file
 from easybuild.tools.github import check_github, find_easybuild_easyconfig, install_github_token
 from easybuild.tools.github import new_pr, merge_pr, update_pr
+from easybuild.tools.hooks import load_hooks, run_hook
 from easybuild.tools.modules import modules_tool
 from easybuild.tools.options import parse_external_modules_metadata, process_software_build_specs, use_color
 from easybuild.tools.robot import check_conflicts, det_robot_path, dry_run, resolve_dependencies, search_easyconfigs

easybuild/tools/filetools.py

@@ -1,4 +1,4 @@
-
+# #
 # Copyright 2009-2017 Ghent University
 #
 # This file is part of EasyBuild,
@@ -1652,81 +1652,3 @@ def diff_files(path1, path2):
     file1_lines = ['%s\n' % l for l in read_file(path1).split('\n')]
     file2_lines = ['%s\n' % l for l in read_file(path2).split('\n')]
     return ''.join(difflib.unified_diff(file1_lines, file2_lines, fromfile=path1, tofile=path2))
-
-
-def load_hooks(hooks_path):
-    """Load defined hooks (if any)."""
-    hooks = []
-    if hooks_path:
-        (hooks_dir, hooks_filename) = os.path.split(hooks_path)
-        (hooks_mod_name, hooks_file_ext) = os.path.splitext(hooks_filename)
-        if hooks_file_ext == '.py':
-            _log.info("Importing hooks implementation from %s...", hooks_path)
-            (fh, pathname, descr) = imp.find_module(hooks_mod_name, [hooks_dir])
-            try:
-                # import module that defines hooks, and collect all functions of which name ends with '_hook'
-                imported_hooks = imp.load_module(hooks_mod_name, fh, pathname, descr)
-                for attr in dir(imported_hooks):
-                    if attr.endswith('_hook'):
-                        hook = getattr(imported_hooks, attr)
-                        if callable(hook):
-                            hooks.append(hook)
-                        else:
-                            _log.debug("Skipping non-callable attribute '%s' when loading hooks", attr)
-                _log.debug("Found hooks: %s", hooks)
-            except ImportError as err:
-                raise EasyBuildError("Failed to import hooks implementation from %s: %s", hooks_path, err)
-        else:
-            raise EasyBuildError("Provided path for hooks implementation should be location of a Python file (*.py)")
-    else:
-        _log.info("No location for hooks implementation provided, no hooks defined")
-
-    return hooks
-
-
-def find_hook(label, known_hooks, pre_step_hook=False, post_step_hook=False):
-    """
-    Find hook with specified label.
-
-    :param label: name of hook
-    :param known_hooks: list of known hooks
-    :param pre_step_hook: indicates whether hook to run is a pre-step hook
-    :param post_step_hook: indicates whether hook to run is a post-step hook
-    """
-    res = None
-
-    if pre_step_hook:
-        hook_prefix = 'pre_'
-    elif post_step_hook:
-        hook_prefix = 'post_'
-    else:
-        hook_prefix = ''
-
-    hook_name = hook_prefix + label + '_hook'
-
-    for hook in known_hooks:
-        if hook.__name__ == hook_name:
-            _log.info("Found %s hook", hook_name)
-            res = hook
-            break
-
-    return res
-
-
-def run_hook(label, known_hooks, pre_step_hook=False, post_step_hook=False, args=None):
-    """
-    Run hook with specified label.
-
-    :param label: name of hook
-    :param known_hooks: list of known hooks
-    :param pre_step_hook: indicates whether hook to run is a pre-step hook
-    :param post_step_hook: indicates whether hook to run is a post-step hook
-    :param args: arguments to pass to hook function
-    """
-    hook = find_hook(label, known_hooks, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook)
-    if hook:
-        if args is None:
-            args = []
-
-        _log.info("Running %s hook (arguments: %s)...", hook.__name__, args)
-        hook(*args)

easybuild/tools/hooks.py

@@ -0,0 +1,113 @@
+# #
+# Copyright 2017-2017 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+# #
+"""
+Hook support.
+
+:author: Kenneth Hoste (Ghent University)
+"""
+import imp
+import os
+from vsc.utils import fancylogger
+
+
+_log = fancylogger.getLogger('hooks', fname=False)
+
+
+def load_hooks(hooks_path):
+    """Load defined hooks (if any)."""
+    hooks = []
+    if hooks_path:
+        (hooks_dir, hooks_filename) = os.path.split(hooks_path)
+        (hooks_mod_name, hooks_file_ext) = os.path.splitext(hooks_filename)
+        if hooks_file_ext == '.py':
+            _log.info("Importing hooks implementation from %s...", hooks_path)
+            (fh, pathname, descr) = imp.find_module(hooks_mod_name, [hooks_dir])
+            try:
+                # import module that defines hooks, and collect all functions of which name ends with '_hook'
+                imported_hooks = imp.load_module(hooks_mod_name, fh, pathname, descr)
+                for attr in dir(imported_hooks):
+                    if attr.endswith('_hook'):
+                        hook = getattr(imported_hooks, attr)
+                        if callable(hook):
+                            hooks.append(hook)
+                        else:
+                            _log.debug("Skipping non-callable attribute '%s' when loading hooks", attr)
+                _log.debug("Found hooks: %s", hooks)
+            except ImportError as err:
+                raise EasyBuildError("Failed to import hooks implementation from %s: %s", hooks_path, err)
+        else:
+            raise EasyBuildError("Provided path for hooks implementation should be location of a Python file (*.py)")
+    else:
+        _log.info("No location for hooks implementation provided, no hooks defined")
+
+    return hooks
+
+
+def find_hook(label, known_hooks, pre_step_hook=False, post_step_hook=False):
+    """
+    Find hook with specified label.
+
+    :param label: name of hook
+    :param known_hooks: list of known hooks
+    :param pre_step_hook: indicates whether hook to run is a pre-step hook
+    :param post_step_hook: indicates whether hook to run is a post-step hook
+    """
+    res = None
+
+    if pre_step_hook:
+        hook_prefix = 'pre_'
+    elif post_step_hook:
+        hook_prefix = 'post_'
+    else:
+        hook_prefix = ''
+
+    hook_name = hook_prefix + label + '_hook'
+
+    for hook in known_hooks:
+        if hook.__name__ == hook_name:
+            _log.info("Found %s hook", hook_name)
+            res = hook
+            break
+
+    return res
+
+
+def run_hook(label, known_hooks, pre_step_hook=False, post_step_hook=False, args=None):
+    """
+    Run hook with specified label.
+
+    :param label: name of hook
+    :param known_hooks: list of known hooks
+    :param pre_step_hook: indicates whether hook to run is a pre-step hook
+    :param post_step_hook: indicates whether hook to run is a post-step hook
+    :param args: arguments to pass to hook function
+    """
+    hook = find_hook(label, known_hooks, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook)
+    if hook:
+        if args is None:
+            args = []
+
+        _log.info("Running %s hook (arguments: %s)...", hook.__name__, args)
+        hook(*args)

test/framework/filetools.py

@@ -1542,74 +1542,6 @@ def test_diff_files(self):
         regex = re.compile('^--- .*/foo\s*\n\+\+\+ .*/bar\s*$', re.M)
         self.assertTrue(regex.search(res), "Pattern '%s' found in: %s" % (regex.pattern, res))
 
-    def test_hooks(self):
-        """Test for functions that support use of hooks."""
-        test_hooks_pymod = os.path.join(self.test_prefix, 'test_hooks.py')
-        test_hooks_pymod_txt = '\n'.join([
-            'def start_hook():',
-            '    print("running start hook")',
-            '',
-            'def foo():',
-            '    print("running foo helper method")',
-            '',
-            'def post_configure_hook(self):',
-            '    print("running post-configure hook")',
-            '    foo()',
-            '',
-            'def pre_install_hook(self):',
-            '    print("running pre-install hook")',
-        ])
-        ft.write_file(test_hooks_pymod, test_hooks_pymod_txt)
-
-        hooks = ft.load_hooks(test_hooks_pymod)
-        self.assertEqual(len(hooks), 3)
-        self.assertEqual(sorted(h.__name__ for h in hooks), ['post_configure_hook', 'pre_install_hook', 'start_hook'])
-        self.assertTrue(all(callable(h) for h in hooks))
-
-        post_configure_hook = [h for h in hooks if h.__name__ == 'post_configure_hook'][0]
-        pre_install_hook = [h for h in hooks if h.__name__ == 'pre_install_hook'][0]
-        start_hook = [h for h in hooks if h.__name__ == 'start_hook'][0]
-
-        self.assertEqual(ft.find_hook('configure', hooks), None)
-        self.assertEqual(ft.find_hook('configure', hooks, pre_step_hook=True), None)
-        self.assertEqual(ft.find_hook('configure', hooks, post_step_hook=True), post_configure_hook)
-
-        self.assertEqual(ft.find_hook('install', hooks), None)
-        self.assertEqual(ft.find_hook('install', hooks, pre_step_hook=True), pre_install_hook)
-        self.assertEqual(ft.find_hook('install', hooks, post_step_hook=True), None)
-
-        self.assertEqual(ft.find_hook('build', hooks), None)
-        self.assertEqual(ft.find_hook('build', hooks, pre_step_hook=True), None)
-        self.assertEqual(ft.find_hook('build', hooks, post_step_hook=True), None)
-
-        self.assertEqual(ft.find_hook('start', hooks), start_hook)
-        self.assertEqual(ft.find_hook('start', hooks, pre_step_hook=True), None)
-        self.assertEqual(ft.find_hook('start', hooks, post_step_hook=True), None)
-
-        self.mock_stdout(True)
-        self.mock_stderr(True)
-        ft.run_hook('start', hooks)
-        ft.run_hook('configure', hooks, pre_step_hook=True, args=[None])
-        ft.run_hook('configure', hooks, post_step_hook=True, args=[None])
-        ft.run_hook('build', hooks, pre_step_hook=True, args=[None])
-        ft.run_hook('build', hooks, post_step_hook=True, args=[None])
-        ft.run_hook('install', hooks, pre_step_hook=True, args=[None])
-        ft.run_hook('install', hooks, post_step_hook=True, args=[None])
-        stdout = self.get_stdout()
-        stderr = self.get_stderr()
-        self.mock_stdout(False)
-        self.mock_stderr(False)
-
-        expected_stdout = '\n'.join([
-            "running start hook",
-            "running post-configure hook",
-            "running foo helper method",
-            "running pre-install hook",
-        ])
-
-        self.assertEqual(stdout.strip(), expected_stdout)
-        self.assertEqual(stderr, '')
-
 
 def suite():
     """ returns all the testcases in this module """