From 5dc338ed43e2182cdfbe65b50a0b6e91275d32f7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 19 Feb 2022 18:18:33 +0100 Subject: [PATCH 001/312] add end-to-end test for running EasyBuild in different Linux distros using containers --- .github/workflows/end2end.yml | 62 +++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/end2end.yml diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml new file mode 100644 index 0000000000..7afd352776 --- /dev/null +++ b/.github/workflows/end2end.yml @@ -0,0 +1,62 @@ +name: End-to-end test of EasyBuild in different distros +on: [push, pull_request] +jobs: + build_publish: + name: End-to-end test + runs-on: ubuntu-latest + strategy: + matrix: + container: + - centos-7.9 + - centos-8.4 + - fedora-35 + - opensuse-15.4 + - rockylinux-8.5 + - ubuntu-20.04 + fail-fast: false + container: + image: ghcr.io/easybuilders/${{ matrix.container }} + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: download and unpack easyblocks and easyconfigs repositories + run: | + cd $HOME + for pkg in easyblocks easyconfigs; do + curl -OL https://github.com/easybuilders/easybuild-${pkg}/archive/develop.tar.gz + tar xfz develop.tar.gz + rm -f develop.tar.gz + done + + - name: End-to-end test of EasyBuild + shell: bash + run: | + export PATH=$PWD:$PATH + export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop + + # initialize environment (for Lmod) + if [ -f /etc/profile.d/modules.sh ]; then + # for Rocky, CentOS 8, Fedora + echo ">>> sourcing /etc/profile.d/modules.sh" + . /etc/profile.d/modules.sh + else + echo ">>> sourcing /etc/profile.d/*lmod*.sh" + . /etc/profile.d/*lmod*.sh + fi + + # tests are run with root privileges, so we need to tell EasyBuild that's OK... + export EASYBUILD_ALLOW_USE_AS_ROOT_AND_ACCEPT_CONSEQUENCES=1 + + cmds=( + "env | sort" + "eb --version" + "eb --show-system-info" + "eb --check-eb-deps" + "eb --show-config" + "eb bzip2-1.0.8.eb --trace --robot" + ) + for cmd in "${cmds[@]}"; do + echo ">>> $cmd" + eval "$cmd" + done From c1d5ca8b2e611604b6efbd4d2f38a8dcb065be3d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 1 Mar 2022 18:03:43 +0100 Subject: [PATCH 002/312] stick to openSUSE 15.3 (stable) rather than using 15.4 (alpha) for now for testing EasyBuild --- .github/workflows/end2end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 7afd352776..7594b79e59 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -10,7 +10,7 @@ jobs: - centos-7.9 - centos-8.4 - fedora-35 - - opensuse-15.4 + - opensuse-15.3 - rockylinux-8.5 - ubuntu-20.04 fail-fast: false From aaf65645f7b2da9dba2ccf1474a7e6d11713dd0b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 21 Dec 2022 16:57:57 +0100 Subject: [PATCH 003/312] Improve error when checksum dict has no entry for a file When the dict didn't contain the filename EB will crash with > Invalid checksum spec 'None', should be a string (MD5) or 2-tuple (type, value). This will now raise a more descriptive error --- easybuild/tools/filetools.py | 9 +++------ test/framework/filetools.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index daa143b46c..f3bc626b2b 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1265,14 +1265,11 @@ def verify_checksum(path, checksums): for checksum in checksums: if isinstance(checksum, dict): - if filename in checksum: + try: # Set this to a string-type checksum checksum = checksum[filename] - elif build_option('enforce_checksums'): - raise EasyBuildError("Missing checksum for %s", filename) - else: - # Set to None and allow to fail elsewhere - checksum = None + except KeyError: + raise EasyBuildError("Missing checksum for %s in %s", filename, checksum) if isinstance(checksum, string_type): # if no checksum type is specified, it is assumed to be MD5 (32 characters) or SHA256 (64 characters) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 32d72c7b83..aa6374e965 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -348,6 +348,14 @@ def test_checksums(self): alt_checksums = ('7167b64b1ca062b9674ffef46f9325db7167b64b1ca062b9674ffef46f9325db', broken_checksums['sha256']) self.assertFalse(ft.verify_checksum(fp, alt_checksums)) + # Check dictionary + alt_checksums = (known_checksums['sha256'],) + self.assertTrue(ft.verify_checksum(fp, {os.path.basename(fp): known_checksums['sha256']})) + faulty_dict = {'wrong-name': known_checksums['sha256']} + self.assertErrorRegex(EasyBuildError, + "Missing checksum for " + os.path.basename(fp) + " in .*wrong-name.*", + ft.verify_checksum, fp, faulty_dict) + # check whether missing checksums are enforced build_options = { 'enforce_checksums': True, @@ -362,6 +370,8 @@ def test_checksums(self): for checksum in [known_checksums[x] for x in ('md5', 'sha256')]: dict_checksum = {os.path.basename(fp): checksum, 'foo': 'baa'} self.assertTrue(ft.verify_checksum(fp, dict_checksum)) + del dict_checksum[os.path.basename(fp)] + self.assertErrorRegex(EasyBuildError, "Missing checksum for", ft.verify_checksum, fp, dict_checksum) def test_common_path_prefix(self): """Test get common path prefix for a list of paths.""" From b9f33eacd44d43d5b6c0a77f58a897ebcf57d3a0 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 3 May 2021 12:02:38 +0200 Subject: [PATCH 004/312] Allow tweaking of ECs from different toolchains This is useful to bulk-install e.g. Python with enable_lto=False set for all ECs. The restriction is only required when changing the toolchain so move the check down and add a simple test. --- easybuild/framework/easyconfig/tweak.py | 18 ++++++----- test/framework/easyconfig.py | 40 ++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 41c688355b..c3e26ab868 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -91,15 +91,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): tweaked_ecs_path, tweaked_ecs_deps_path = None, None if targetdirs is not None: tweaked_ecs_path, tweaked_ecs_deps_path = targetdirs - # make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble) - toolchains = nub(['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs]) - if len(toolchains) > 1: - raise EasyBuildError("Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s", - toolchains) - # Toolchain is unique, let's store it - source_toolchain = easyconfigs[-1]['ec']['toolchain'] modifying_toolchains_or_deps = False - target_toolchain = {} src_to_dst_tc_mapping = {} revert_to_regex = False @@ -117,6 +109,16 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): revert_to_regex = True if not revert_to_regex: + # make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble) + toolchains = nub(['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs]) + if len(toolchains) > 1: + raise EasyBuildError("Multiple toolchains featured in easyconfigs, " + "--try-X not supported in that case: %s", + toolchains) + # Toolchain is unique, let's store it + source_toolchain = easyconfigs[-1]['ec']['toolchain'] + target_toolchain = {} + # we're doing something that involves the toolchain hierarchy; # obtain full dependency graph for specified easyconfigs; # easyconfigs will be ordered 'top-to-bottom' (toolchains and dependencies appearing first) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index f09a0896a6..dfcd956f02 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -57,10 +57,10 @@ from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig from easybuild.framework.easyconfig.templates import template_constant_dict, to_template_str from easybuild.framework.easyconfig.style import check_easyconfigs_style -from easybuild.framework.easyconfig.tools import categorize_files_by_type, check_sha256_checksums, dep_graph -from easybuild.framework.easyconfig.tools import det_copy_ec_specs, find_related_easyconfigs, get_paths_for +from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, categorize_files_by_type, check_sha256_checksums +from easybuild.framework.easyconfig.tools import dep_graph, det_copy_ec_specs, find_related_easyconfigs, get_paths_for from easybuild.framework.easyconfig.tools import parse_easyconfigs -from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak_one +from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak, tweak_one from easybuild.framework.extension import resolve_exts_filter_template from easybuild.toolchains.system import SystemToolchain from easybuild.tools.build_log import EasyBuildError @@ -73,7 +73,7 @@ from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.options import parse_external_modules_metadata from easybuild.tools.py2vs3 import OrderedDict, reload -from easybuild.tools.robot import resolve_dependencies +from easybuild.tools.robot import det_robot_path, resolve_dependencies from easybuild.tools.systemtools import AARCH64, KNOWN_ARCH_CONSTANTS, POWER, X86_64 from easybuild.tools.systemtools import get_cpu_architecture, get_shared_lib_ext, get_os_name, get_os_version @@ -728,6 +728,38 @@ def test_tweaking(self): # cleanup os.remove(tweaked_fn) + def test_tweak_multiple_tcs(self): + """Test that tweaking variables of ECs from multiple toolchains works""" + test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + + # Create directories to store the tweaked easyconfigs + tweaked_ecs_paths, pr_path = alt_easyconfig_paths(self.test_prefix, tweaked_ecs=True) + robot_path = det_robot_path([test_easyconfigs], tweaked_ecs_paths, pr_path, auto_robot=True) + + init_config(build_options={ + 'valid_module_classes': module_classes(), + 'robot_path': robot_path, + 'check_osdeps': False, + }) + + # Allow tweaking of non-toolchain values for multiple ECs of different toolchains + untweaked_openmpi_1 = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-4.6.4.eb') + untweaked_openmpi_2 = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-3.1.1-GCC-7.3.0-2.30.eb') + easyconfigs, _ = parse_easyconfigs([(untweaked_openmpi_1, False), (untweaked_openmpi_2, False)]) + tweak_specs = {'moduleclass': 'debugger'} + easyconfigs = tweak(easyconfigs, tweak_specs, self.modtool, targetdirs=tweaked_ecs_paths) + # Check that all expected tweaked easyconfigs exists + tweaked_openmpi_1 = os.path.join(tweaked_ecs_paths[0], os.path.basename(untweaked_openmpi_1)) + tweaked_openmpi_2 = os.path.join(tweaked_ecs_paths[0], os.path.basename(untweaked_openmpi_2)) + self.assertTrue(os.path.isfile(tweaked_openmpi_1)) + self.assertTrue(os.path.isfile(tweaked_openmpi_2)) + tweaked_openmpi_content_1 = read_file(tweaked_openmpi_1) + tweaked_openmpi_content_2 = read_file(tweaked_openmpi_2) + self.assertTrue('moduleclass = "debugger"' in tweaked_openmpi_content_1, + "Tweaked value not found in " + tweaked_openmpi_content_1) + self.assertTrue('moduleclass = "debugger"' in tweaked_openmpi_content_2, + "Tweaked value not found in " + tweaked_openmpi_content_2) + def test_installversion(self): """Test generation of install version.""" From f140a328ba6a2398931e92ce5858d36ac0668e20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Kr=C3=A1l?= Date: Fri, 19 May 2023 15:22:45 +0200 Subject: [PATCH 005/312] fix error message when checksums set to [None] --- easybuild/framework/easyblock.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index df73e28db4..33d96166e0 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -395,13 +395,13 @@ def get_checksums_from_json(self, always_read=False): :param always_read: always read the checksums.json file, even if it has been read before """ if always_read or self.json_checksums is None: - try: - path = self.obtain_file("checksums.json", no_download=True) + path = self.obtain_file("checksums.json", no_download=True, warning_only=True) + if path is not None: self.log.info("Loading checksums from file %s", path) json_txt = read_file(path) self.json_checksums = json.loads(json_txt) - # if the file can't be found, return an empty dict - except EasyBuildError: + else: + # if the file can't be found, return an empty dict self.json_checksums = {} return self.json_checksums @@ -736,7 +736,8 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): return exts_sources def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False, - git_config=None, no_download=False, download_instructions=None, alt_location=None): + git_config=None, no_download=False, download_instructions=None, alt_location=None, + warning_only=False): """ Locate the file with the given name - searches in different subdirectories of source path @@ -789,7 +790,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No return fullpath except IOError as err: - raise EasyBuildError("Downloading file %s from url %s to %s failed: %s", filename, url, fullpath, err) + if not warning_only: + raise EasyBuildError("Downloading file %s " + "from url %s to %s failed: %s", filename, url, fullpath, err) + else: + self.log.warning("Downloading file %s " + "from url %s to %s failed: %s", filename, url, fullpath, err) + return None else: # try and find file in various locations @@ -866,8 +873,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No self.dry_run_msg(" * %s (MISSING)", filename) return filename else: - raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... " + if not warning_only: + raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... " + "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) + else: + self.log.warning("Couldn't find file %s anywhere, and downloading it is disabled... " "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) + return None elif git_config: return get_source_tarball_from_git(filename, targetdir, git_config) else: @@ -959,7 +971,11 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No error_msg += "and downloading it didn't work either... " error_msg += "Paths attempted (in order): %s " % failedpaths_msg - raise EasyBuildError(error_msg, filename) + if not warning_only: + raise EasyBuildError(error_msg, filename) + else: + self.log.warning(error_msg, filename) + return None # # GETTER/SETTER UTILITY FUNCTIONS From ea926f1e1a998534532058242e5f55b2b427ba85 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 21 Dec 2022 16:57:57 +0100 Subject: [PATCH 006/312] Improve error when checksum dict has no entry for a file When the dict didn't contain the filename EB will crash with > Invalid checksum spec 'None', should be a string (MD5) or 2-tuple (type, value). This will now raise a more descriptive error --- easybuild/tools/filetools.py | 9 +++------ test/framework/filetools.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index daa143b46c..f3bc626b2b 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1265,14 +1265,11 @@ def verify_checksum(path, checksums): for checksum in checksums: if isinstance(checksum, dict): - if filename in checksum: + try: # Set this to a string-type checksum checksum = checksum[filename] - elif build_option('enforce_checksums'): - raise EasyBuildError("Missing checksum for %s", filename) - else: - # Set to None and allow to fail elsewhere - checksum = None + except KeyError: + raise EasyBuildError("Missing checksum for %s in %s", filename, checksum) if isinstance(checksum, string_type): # if no checksum type is specified, it is assumed to be MD5 (32 characters) or SHA256 (64 characters) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 32d72c7b83..aa6374e965 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -348,6 +348,14 @@ def test_checksums(self): alt_checksums = ('7167b64b1ca062b9674ffef46f9325db7167b64b1ca062b9674ffef46f9325db', broken_checksums['sha256']) self.assertFalse(ft.verify_checksum(fp, alt_checksums)) + # Check dictionary + alt_checksums = (known_checksums['sha256'],) + self.assertTrue(ft.verify_checksum(fp, {os.path.basename(fp): known_checksums['sha256']})) + faulty_dict = {'wrong-name': known_checksums['sha256']} + self.assertErrorRegex(EasyBuildError, + "Missing checksum for " + os.path.basename(fp) + " in .*wrong-name.*", + ft.verify_checksum, fp, faulty_dict) + # check whether missing checksums are enforced build_options = { 'enforce_checksums': True, @@ -362,6 +370,8 @@ def test_checksums(self): for checksum in [known_checksums[x] for x in ('md5', 'sha256')]: dict_checksum = {os.path.basename(fp): checksum, 'foo': 'baa'} self.assertTrue(ft.verify_checksum(fp, dict_checksum)) + del dict_checksum[os.path.basename(fp)] + self.assertErrorRegex(EasyBuildError, "Missing checksum for", ft.verify_checksum, fp, dict_checksum) def test_common_path_prefix(self): """Test get common path prefix for a list of paths.""" From 9e0c6808d6bbe1c3db6e1f2011819149a643b29e Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 10 Jun 2023 10:53:08 +0100 Subject: [PATCH 007/312] remove deprecated desc option for simple_option --- easybuild/base/generaloption.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index 3e9788ac44..aff26d15d0 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -1711,7 +1711,7 @@ class SimpleOption(GeneralOption): PARSER = SimpleOptionParser SETROOTLOGGER = True - def __init__(self, go_dict=None, descr=None, short_groupdescr=None, long_groupdescr=None, config_files=None): + def __init__(self, go_dict=None, short_groupdescr=None, long_groupdescr=None, config_files=None): """Initialisation :param go_dict: General Option option dict :param short_groupdescr: short description of main options @@ -1740,18 +1740,13 @@ def __init__(self, go_dict=None, descr=None, short_groupdescr=None, long_groupde super(SimpleOption, self).__init__(**kwargs) - if descr is not None: - # TODO: as there is no easy/clean way to access the version of the vsc-base package, - # this is equivalent to a warning - self.log.deprecated('SimpleOption descr argument', '2.5.0', '3.0.0') - def main_options(self): if self.go_dict is not None: prefix = None self.add_group_parser(self.go_dict, self.descr, prefix=prefix) -def simple_option(go_dict=None, descr=None, short_groupdescr=None, long_groupdescr=None, config_files=None): +def simple_option(go_dict=None, short_groupdescr=None, long_groupdescr=None, config_files=None): """A function that returns a single level GeneralOption option parser :param go_dict: General Option option dict @@ -1765,5 +1760,5 @@ def simple_option(go_dict=None, descr=None, short_groupdescr=None, long_groupdes the generated help will include the docstring """ - return SimpleOption(go_dict=go_dict, descr=descr, short_groupdescr=short_groupdescr, - long_groupdescr=long_groupdescr, config_files=config_files) + return SimpleOption(go_dict=go_dict, short_groupdescr=short_groupdescr, long_groupdescr=long_groupdescr, + config_files=config_files) From 4981b141b1376ff6dc940e5acf91da442b0d19bd Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 10 Jun 2023 10:57:24 +0100 Subject: [PATCH 008/312] remove deprecated Toolchain.add_dependencies --- easybuild/tools/toolchain/toolchain.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 03edfa7359..bb4b9e34c5 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -550,16 +550,6 @@ def _check_dependencies(self, dependencies): return deps - def add_dependencies(self, dependencies): - """ - [DEPRECATED] Verify if the given dependencies exist, and return them. - - This method is deprecated. - You should pass the dependencies to the 'prepare' method instead, via the 'deps' named argument. - """ - self.log.deprecated("use of 'Toolchain.add_dependencies' method", '4.0') - self.dependencies = self._check_dependencies(dependencies) - def is_required(self, name): """Determine whether this is a required toolchain element.""" # default: assume every element is required From b243c9e35cfe9a6797bacbb27e45e2de82986624 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 10 Jun 2023 11:24:45 +0100 Subject: [PATCH 009/312] remove deprecated functionality from filetools --- easybuild/tools/filetools.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 8848edcc89..1261cd1df4 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1727,7 +1727,7 @@ def convert_name(name, upper=False): def adjust_permissions(provided_path, permission_bits, add=True, onlyfiles=False, onlydirs=False, recursive=True, - group_id=None, relative=True, ignore_errors=False, skip_symlinks=None): + group_id=None, relative=True, ignore_errors=False): """ Change permissions for specified path, using specified permission bits @@ -1744,11 +1744,6 @@ def adjust_permissions(provided_path, permission_bits, add=True, onlyfiles=False and directories (if onlyfiles is False) in path """ - if skip_symlinks is not None: - depr_msg = "Use of 'skip_symlinks' argument for 'adjust_permissions' is deprecated " - depr_msg += "(symlinks are never followed anymore)" - _log.deprecated(depr_msg, '4.0') - provided_path = os.path.abspath(provided_path) if recursive: @@ -2078,13 +2073,6 @@ def path_matches(path, paths): return False -def rmtree2(path, n=3): - """Wrapper around shutil.rmtree to make it more robust when used on NFS mounted file systems.""" - - _log.deprecated("Use 'remove_dir' rather than 'rmtree2'", '5.0') - remove_dir(path) - - def find_backup_name_candidate(src_file): """Returns a non-existing file to be used as destination for backup files""" @@ -2191,11 +2179,6 @@ def cleanup(logfile, tempdir, testing, silent=False): print_msg(msg, log=None, silent=testing or silent) -def copytree(src, dst, symlinks=False, ignore=None): - """DEPRECATED and removed. Use copy_dir""" - _log.deprecated("Use 'copy_dir' rather than 'copytree'", '4.0') - - def encode_string(name): """ This encoding function handles funky software names ad infinitum, like: From c3acae5c2eb1cb9f7d81c2cbfaf2361cc8daca7d Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sun, 11 Jun 2023 09:16:06 +0100 Subject: [PATCH 010/312] remove deprecated log_error option from which command --- easybuild/tools/filetools.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 8848edcc89..dc40d8e41d 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -484,7 +484,7 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced return base_dir -def which(cmd, retain_all=False, check_perms=True, log_ok=True, log_error=None, on_error=None): +def which(cmd, retain_all=False, check_perms=True, log_ok=True, log_error=None, on_error=WARN): """ Return (first) path in $PATH for specified command, or None if command is not found @@ -493,17 +493,6 @@ def which(cmd, retain_all=False, check_perms=True, log_ok=True, log_error=None, :param log_ok: Log an info message where the command has been found (if any) :param on_error: What to do if the command was not found, default: WARN. Possible values: IGNORE, WARN, ERROR """ - if log_error is not None: - _log.deprecated("'log_error' named argument in which function has been replaced by 'on_error'", '5.0') - # If set, make sure on_error is at least WARN - if log_error and on_error == IGNORE: - on_error = WARN - elif not log_error and on_error is None: # If set to False, use IGNORE unless on_error is also set - on_error = IGNORE - # Set default - # TODO: After removal of log_error from the parameters, on_error=WARN can be used instead of this - if on_error is None: - on_error = WARN if on_error not in (IGNORE, WARN, ERROR): raise EasyBuildError("Invalid value for 'on_error': %s", on_error) From 2e2e32810f8fe03db0733ae26f309c867cb5a590 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sun, 11 Jun 2023 09:19:09 +0100 Subject: [PATCH 011/312] actually remove the option --- easybuild/tools/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index dc40d8e41d..17a13b75a5 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -484,7 +484,7 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced return base_dir -def which(cmd, retain_all=False, check_perms=True, log_ok=True, log_error=None, on_error=WARN): +def which(cmd, retain_all=False, check_perms=True, log_ok=True, on_error=WARN): """ Return (first) path in $PATH for specified command, or None if command is not found From 71f620ee0f5e6a52d7432b31d64d56f598d9b0ea Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sun, 11 Jun 2023 09:26:06 +0100 Subject: [PATCH 012/312] remove deprecated skip_lower option from template_constant_dict --- easybuild/framework/easyconfig/templates.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 728e5fcf52..04ffdff1e7 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -181,13 +181,10 @@ # versionmajor, versionminor, versionmajorminor (eg '.'.join(version.split('.')[:2])) ) -def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None): +def template_constant_dict(config, ignore=None, toolchain=None): """Create a dict for templating the values in the easyconfigs. - config is a dict with the structure of EasyConfig._config """ - if skip_lower is not None: - _log.deprecated("Use of 'skip_lower' named argument for template_constant_dict has no effect anymore", '4.0') - # TODO find better name # ignore if ignore is None: From dbf5e44c31d49fd0210b25d1c5fdd3a70b5df3a4 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sun, 11 Jun 2023 09:42:45 +0100 Subject: [PATCH 013/312] remove deprecated options from easyconfig.py --- easybuild/framework/easyconfig/easyconfig.py | 25 +------------------- 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index b0704ca11a..c65b4dbe09 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -403,20 +403,6 @@ def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False): return toolchain_hierarchy -@contextmanager -def disable_templating(ec): - """Temporarily disable templating on the given EasyConfig - - Usage: - with disable_templating(ec): - # Do what you want without templating - # Templating set to previous value - """ - _log.deprecated("disable_templating(ec) was replaced by ec.disable_templating()", '5.0') - with ec.disable_templating() as old_value: - yield old_value - - class EasyConfig(object): """ Class which handles loading, reading, validation of easyconfigs @@ -1857,19 +1843,10 @@ def det_installversion(version, toolchain_name, toolchain_version, prefix, suffi _log.nosupport('Use det_full_ec_version from easybuild.tools.module_generator instead of %s' % old_fn, '2.0') -def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error_on_missing_easyblock=None, **kwargs): +def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error_on_missing_easyblock=True, **kwargs): """ Get class for a particular easyblock (or use default) """ - if 'default_fallback' in kwargs: - msg = "Named argument 'default_fallback' for get_easyblock_class is deprecated, " - msg += "use 'error_on_missing_easyblock' instead" - _log.deprecated(msg, '4.0') - if error_on_missing_easyblock is None: - error_on_missing_easyblock = kwargs['default_fallback'] - elif error_on_missing_easyblock is None: - error_on_missing_easyblock = True - cls = None try: if easyblock: From 8b4cf52920d5343640e9b38f6f0bda336f6fd929 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sun, 11 Jun 2023 09:50:57 +0100 Subject: [PATCH 014/312] removed deprecated mod_exists_regex_template to exist --- easybuild/tools/modules.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 9037be6ea8..8fd2fc9d09 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -548,18 +548,14 @@ def module_wrapper_exists(self, mod_name, modulerc_fn='.modulerc', mod_wrapper_r return wrapped_mod - def exist(self, mod_names, mod_exists_regex_template=None, skip_avail=False, maybe_partial=True): + def exist(self, mod_names, skip_avail=False, maybe_partial=True): """ Check if modules with specified names exists. :param mod_names: list of module names - :param mod_exists_regex_template: DEPRECATED and unused :param skip_avail: skip checking through 'module avail', only check via 'module show' :param maybe_partial: indicates if the module name may be a partial module name """ - if mod_exists_regex_template is not None: - self.log.deprecated('mod_exists_regex_template is no longer used', '5.0') - def mod_exists_via_show(mod_name): """ Helper function to check whether specified module name exists through 'module show'. From 2c720d8320eeaf5d2ba53ded9a17138d587ca5d0 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sun, 11 Jun 2023 10:07:43 +0100 Subject: [PATCH 015/312] remove test for removed option --- test/framework/easyconfig.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 7c83a15f75..94b4ba66ae 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1528,17 +1528,6 @@ def test_get_easyblock_class(self): self.assertErrorRegex(EasyBuildError, "Failed to import EB_TOY", get_easyblock_class, None, name='TOY') self.assertEqual(get_easyblock_class(None, name='TOY', error_on_failed_import=False), None) - # also test deprecated default_fallback named argument - self.assertErrorRegex(EasyBuildError, "DEPRECATED", get_easyblock_class, None, name='gzip', - default_fallback=False) - - orig_value = easybuild.tools.build_log.CURRENT_VERSION - easybuild.tools.build_log.CURRENT_VERSION = '3.9' - self.mock_stderr(True) - self.assertEqual(get_easyblock_class(None, name='gzip', default_fallback=False), None) - self.mock_stderr(False) - easybuild.tools.build_log.CURRENT_VERSION = orig_value - def test_letter_dir(self): """Test letter_dir_for function.""" test_cases = { From 3da393aa2ce8cb82af3b10c0bdda139461899873 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 20 Jan 2023 13:21:52 +0100 Subject: [PATCH 016/312] reduce number of CI jobs We run into rate limits installing the dependencies. We can reduce the number of CI jobs to almost half by running the tests in a loop for each supported module syntax. --- .github/workflows/unit_tests.yml | 70 +++++++++++++------------------- 1 file changed, 29 insertions(+), 41 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 5ff4a24521..d2f8ec0ae9 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -34,47 +34,25 @@ jobs: - ${{needs.setup.outputs.modulesTcl}} - ${{needs.setup.outputs.modules3}} - ${{needs.setup.outputs.modules4}} - module_syntax: [Lua, Tcl] lc_all: [""] - # don't test with Lua module syntax (only supported in Lmod) - exclude: - - modules_tool: ${{needs.setup.outputs.modulesTcl}} - module_syntax: Lua - - modules_tool: ${{needs.setup.outputs.modules3}} - module_syntax: Lua - - modules_tool: ${{needs.setup.outputs.modules4}} - module_syntax: Lua include: - # Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax) + # Test different Python 3 versions with Lmod 8.x - python: 3.5 modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Lua - python: 3.7 modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Lua - python: 3.8 modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Lua - - python: 3.8 - modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Tcl - python: 3.9 modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Lua - python: '3.10' modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Lua - - python: '3.11' - modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Lua - python: '3.11' modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Tcl # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7) - python: 3.6 modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Lua lc_all: C fail-fast: false steps: @@ -129,7 +107,7 @@ jobs: # and only when testing with Lua as module syntax, # to avoid hitting GitHub rate limit; # tests that require a GitHub token are skipped automatically when no GitHub token is available - if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]] && [[ "${{matrix.module_syntax}}" == 'Lua' ]]; then + if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]]; then if [ ! -z $GITHUB_TOKEN ]; then if [ "x${{matrix.python}}" == 'x2.6' ]; then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())"; @@ -172,8 +150,6 @@ jobs: - name: run test suite env: EB_VERBOSE: 1 - EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}} - TEST_EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}} LC_ALL: ${{matrix.lc_all}} run: | # run tests *outside* of checked out easybuild-framework directory, @@ -199,18 +175,30 @@ jobs: export EASYBUILD_MODULES_TOOL=Lmod fi export TEST_EASYBUILD_MODULES_TOOL=$EASYBUILD_MODULES_TOOL - eb --show-config - # gather some useful info on test system - eb --show-system-info - # check GitHub configuration - eb --check-github --github-user=easybuild_test - # create file owned by root but writable by anyone (used by test_copy_file) - sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt - sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt - # run test suite - python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log - # try and make sure output of running tests is clean (no printed messages/warnings) - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test" - # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches) - PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) - test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1) + + # Run tests with LUA and Tcl module syntax (where supported) + for module_syntax in Lua Tcl; do + # Only Lmod supports Lua + if [[ "$module_syntax" == "Lua" ]] && [[ "$EASYBUILD_MODULES_TOOL" != "Lmod" ]]; then + continue + fi + printf '\n\n=====================> Using $module_syntax module syntax <=====================\n\n' + export EASYBUILD_MODULE_SYNTAX="$module_syntax" + export TEST_EASYBUILD_MODULE_SYNTAX="$EASYBUILD_MODULE_SYNTAX" + + eb --show-config + # gather some useful info on test system + eb --show-system-info + # check GitHub configuration + eb --check-github --github-user=easybuild_test + # create file owned by root but writable by anyone (used by test_copy_file) + sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt + sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt + # run test suite + python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log + # try and make sure output of running tests is clean (no printed messages/warnings) + IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test" + # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches) + PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) + test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1) + done From ccc6ef7644969b7117224aa5b6f58e47ad34ca66 Mon Sep 17 00:00:00 2001 From: "Xavier Elwell (Advanced Research Computing)" Date: Tue, 25 Jul 2023 11:04:49 +0100 Subject: [PATCH 017/312] Add cancel and fail hooks --- easybuild/main.py | 5 ++++- easybuild/tools/hooks.py | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 319c80b5ae..32a3e922ef 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -734,5 +734,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): main() except EasyBuildError as err: print_error(err.msg) - except KeyboardInterrupt as err: + except KeyboardInterrupt as err: + eb_go, cfg_settings = set_up_configuration(args=None, logfile=None, testing=None) + options, orig_paths = eb_go.options, eb_go.args + run_hook('cancel', load_hooks(options.hooks)) print_error("Cancelled by user: %s" % err) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 3c21d4e104..f072a2bcd4 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -65,6 +65,9 @@ MODULE_WRITE = 'module_write' END = 'end' +FAIL = 'fail' +CANCEL = 'cancel' + PRE_PREF = 'pre_' POST_PREF = 'post_' HOOK_SUFF = '_hook' @@ -98,6 +101,8 @@ ] + [p + x for x in STEP_NAMES[STEP_NAMES.index(MODULE_STEP)+1:] for p in [PRE_PREF, POST_PREF]] + [ END, + FAIL, + CANCEL, ] KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES] From 3eee39508317195cae141fefb076ac32f5a8fecf Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 25 Jul 2023 13:30:00 +0100 Subject: [PATCH 018/312] Run hooks in main with arguments --- easybuild/main.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 32a3e922ef..a3186b68cd 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -69,7 +69,7 @@ from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, merge_pr, new_branch_github, new_pr from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr -from easybuild.tools.hooks import START, END, load_hooks, run_hook +from easybuild.tools.hooks import START, END, CANCEL, FAIL, load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import opts_dict_to_eb_opts, set_up_configuration, use_color from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm @@ -730,12 +730,13 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if __name__ == "__main__": + eb_go, _ = set_up_configuration(args=None, logfile=None, testing=None) + hooks = load_hooks(eb_go.options.hooks) try: main() except EasyBuildError as err: + run_hook(FAIL, hooks, args=[err]) print_error(err.msg) - except KeyboardInterrupt as err: - eb_go, cfg_settings = set_up_configuration(args=None, logfile=None, testing=None) - options, orig_paths = eb_go.options, eb_go.args - run_hook('cancel', load_hooks(options.hooks)) + except KeyboardInterrupt as err: + run_hook(CANCEL, hooks, args=[err]) print_error("Cancelled by user: %s" % err) From a80ac7f57f8b6728e104ff8d3326f63bcd94668a Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 25 Jul 2023 14:17:10 +0100 Subject: [PATCH 019/312] Init eb_go with parse_options() --- easybuild/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index a3186b68cd..e93af02890 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -71,7 +71,7 @@ from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr from easybuild.tools.hooks import START, END, CANCEL, FAIL, load_hooks, run_hook from easybuild.tools.modules import modules_tool -from easybuild.tools.options import opts_dict_to_eb_opts, set_up_configuration, use_color +from easybuild.tools.options import opts_dict_to_eb_opts, parse_options, set_up_configuration, use_color from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs @@ -730,7 +730,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if __name__ == "__main__": - eb_go, _ = set_up_configuration(args=None, logfile=None, testing=None) + eb_go = parse_options() hooks = load_hooks(eb_go.options.hooks) try: main() From 65111fb413ec48a142ec433bcac21b953bf48720 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 3 Aug 2023 13:46:16 +0200 Subject: [PATCH 020/312] Ignore request for external module (meta)data when no modules tool is active E.g. for `--fetch` the modules tool is explicitly set to `None`/`NoModulesTool` to allow running on e.g. laptops to fetch sources. This however fails for external module dependencies in which it tries to resolve them using the modules tool which fails with `NotImplementedError`. Simply ignore the request in that case as-if the module metadata wasn't found. Fixes #4298 --- easybuild/framework/easyconfig/easyconfig.py | 5 +++- test/framework/options.py | 25 +++++++++++--------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index f4c4be464e..55fe3c8578 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -73,7 +73,7 @@ from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name -from easybuild.tools.modules import modules_tool +from easybuild.tools.modules import modules_tool, NoModulesTool from easybuild.tools.py2vs3 import OrderedDict, create_base_metaclass, string_type from easybuild.tools.systemtools import check_os_dependency, pick_dep_version from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain @@ -1306,6 +1306,9 @@ def probe_external_module_metadata(self, mod_name, existing_metadata=None): :param existing_metadata: already available metadata for this external module (if any) """ res = {} + if isinstance(self.modules_tool, NoModulesTool): + self.log.debug('Ignoring request for external module data for %s as no modules tool is active', mod_name) + return res if existing_metadata is None: existing_metadata = {} diff --git a/test/framework/options.py b/test/framework/options.py index 106f951374..265e154e55 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5073,19 +5073,22 @@ def test_fetch(self): lock_path = os.path.join(self.test_installpath, 'software', '.locks', lock_fn) mkdir(lock_path, parents=True) - args = ['toy-0.0.eb', '--fetch'] - stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True, testing=False) + # Run for a "regular" EC and one with an external module dependency + # which might trip up the dependency resolution (see #4298) + for ec in ('toy-0.0.eb', 'toy-0.0-deps.eb'): + args = [ec, '--fetch'] + stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True, testing=False) - patterns = [ - r"^== fetching files\.\.\.$", - r"^== COMPLETED: Installation STOPPED successfully \(took .* secs?\)$", - ] - for pattern in patterns: - regex = re.compile(pattern, re.M) - self.assertTrue(regex.search(stdout), "Pattern '%s' not found in: %s" % (regex.pattern, stdout)) + patterns = [ + r"^== fetching files\.\.\.$", + r"^== COMPLETED: Installation STOPPED successfully \(took .* secs?\)$", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' not found in: %s" % (regex.pattern, stdout)) - regex = re.compile(r"^== creating build dir, resetting environment\.\.\.$") - self.assertFalse(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + regex = re.compile(r"^== creating build dir, resetting environment\.\.\.$") + self.assertFalse(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) def test_parse_external_modules_metadata(self): """Test parse_external_modules_metadata function.""" From 2ef4cf6239eab39c97db787f4409423ae9db695f Mon Sep 17 00:00:00 2001 From: "Xavier Elwell (Advanced Research Computing)" Date: Mon, 7 Aug 2023 11:27:49 +0100 Subject: [PATCH 021/312] Merge with main --- easybuild/scripts/EasyBuild-reinstall-dev.sh | 0 easybuild/scripts/bootstrap_eb.py | 0 easybuild/scripts/clean_gists.py | 0 easybuild/scripts/createSubmoduleDeps.sh | 0 easybuild/scripts/findPythonDeps.py | 0 easybuild/scripts/findUpdatedEcs.sh | 0 easybuild/scripts/fix_docs.py | 0 easybuild/scripts/install-EasyBuild-develop.sh | 0 easybuild/scripts/install-EasyBuild-sprint.sh | 0 easybuild/scripts/install_eb_dep.sh | 0 easybuild/scripts/mk_tmpl_easyblock_for.py | 0 easybuild/scripts/rpath_args.py | 0 eb | 0 test/framework/suite.py | 0 14 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 easybuild/scripts/EasyBuild-reinstall-dev.sh mode change 100755 => 100644 easybuild/scripts/bootstrap_eb.py mode change 100755 => 100644 easybuild/scripts/clean_gists.py mode change 100755 => 100644 easybuild/scripts/createSubmoduleDeps.sh mode change 100755 => 100644 easybuild/scripts/findPythonDeps.py mode change 100755 => 100644 easybuild/scripts/findUpdatedEcs.sh mode change 100755 => 100644 easybuild/scripts/fix_docs.py mode change 100755 => 100644 easybuild/scripts/install-EasyBuild-develop.sh mode change 100755 => 100644 easybuild/scripts/install-EasyBuild-sprint.sh mode change 100755 => 100644 easybuild/scripts/install_eb_dep.sh mode change 100755 => 100644 easybuild/scripts/mk_tmpl_easyblock_for.py mode change 100755 => 100644 easybuild/scripts/rpath_args.py mode change 100755 => 100644 eb mode change 100755 => 100644 test/framework/suite.py diff --git a/easybuild/scripts/EasyBuild-reinstall-dev.sh b/easybuild/scripts/EasyBuild-reinstall-dev.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/createSubmoduleDeps.sh b/easybuild/scripts/createSubmoduleDeps.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/findUpdatedEcs.sh b/easybuild/scripts/findUpdatedEcs.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/fix_docs.py b/easybuild/scripts/fix_docs.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/install-EasyBuild-develop.sh b/easybuild/scripts/install-EasyBuild-develop.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/install-EasyBuild-sprint.sh b/easybuild/scripts/install-EasyBuild-sprint.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/install_eb_dep.sh b/easybuild/scripts/install_eb_dep.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py old mode 100755 new mode 100644 diff --git a/eb b/eb old mode 100755 new mode 100644 diff --git a/test/framework/suite.py b/test/framework/suite.py old mode 100755 new mode 100644 From 014fd42092bb5309b5117adf9d8be1fb9f351843 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 8 Aug 2023 10:41:29 +0100 Subject: [PATCH 022/312] Add crash hook --- easybuild/main.py | 5 ++++- easybuild/tools/hooks.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index e93af02890..b5c2f9a01e 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -69,7 +69,7 @@ from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, merge_pr, new_branch_github, new_pr from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr -from easybuild.tools.hooks import START, END, CANCEL, FAIL, load_hooks, run_hook +from easybuild.tools.hooks import CRASH, START, END, CANCEL, FAIL, load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import opts_dict_to_eb_opts, parse_options, set_up_configuration, use_color from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm @@ -740,3 +740,6 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): except KeyboardInterrupt as err: run_hook(CANCEL, hooks, args=[err]) print_error("Cancelled by user: %s" % err) + except Exception as err: + run_hook(CRASH, hooks, args=[err]) + print_error("Encountered an unrecoverable error: %s" % err) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index f072a2bcd4..ee1af63b6c 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -67,6 +67,7 @@ FAIL = 'fail' CANCEL = 'cancel' +CRASH = 'crash' PRE_PREF = 'pre_' POST_PREF = 'post_' @@ -103,6 +104,7 @@ END, FAIL, CANCEL, + CRASH, ] KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES] From 1a589dce3c7a9801a1717cee3e2d73faeb1c10e1 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 8 Aug 2023 10:47:40 +0100 Subject: [PATCH 023/312] Remove trailing whitespace --- easybuild/tools/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index ee1af63b6c..de07866c16 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -102,7 +102,7 @@ ] + [p + x for x in STEP_NAMES[STEP_NAMES.index(MODULE_STEP)+1:] for p in [PRE_PREF, POST_PREF]] + [ END, - FAIL, + FAIL, CANCEL, CRASH, ] From 565c11311524ab1ad172df01cf2fdb0d7999b6bd Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 8 Aug 2023 10:51:03 +0100 Subject: [PATCH 024/312] Revert file permissions --- easybuild/scripts/EasyBuild-reinstall-dev.sh | 0 easybuild/scripts/bootstrap_eb.py | 0 easybuild/scripts/clean_gists.py | 0 easybuild/scripts/createSubmoduleDeps.sh | 0 easybuild/scripts/findPythonDeps.py | 0 easybuild/scripts/findUpdatedEcs.sh | 0 easybuild/scripts/fix_docs.py | 0 easybuild/scripts/install-EasyBuild-develop.sh | 0 easybuild/scripts/install-EasyBuild-sprint.sh | 0 easybuild/scripts/install_eb_dep.sh | 0 easybuild/scripts/mk_tmpl_easyblock_for.py | 0 easybuild/scripts/rpath_args.py | 0 eb | 0 test/framework/suite.py | 0 14 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 easybuild/scripts/EasyBuild-reinstall-dev.sh mode change 100644 => 100755 easybuild/scripts/bootstrap_eb.py mode change 100644 => 100755 easybuild/scripts/clean_gists.py mode change 100644 => 100755 easybuild/scripts/createSubmoduleDeps.sh mode change 100644 => 100755 easybuild/scripts/findPythonDeps.py mode change 100644 => 100755 easybuild/scripts/findUpdatedEcs.sh mode change 100644 => 100755 easybuild/scripts/fix_docs.py mode change 100644 => 100755 easybuild/scripts/install-EasyBuild-develop.sh mode change 100644 => 100755 easybuild/scripts/install-EasyBuild-sprint.sh mode change 100644 => 100755 easybuild/scripts/install_eb_dep.sh mode change 100644 => 100755 easybuild/scripts/mk_tmpl_easyblock_for.py mode change 100644 => 100755 easybuild/scripts/rpath_args.py mode change 100644 => 100755 eb mode change 100644 => 100755 test/framework/suite.py diff --git a/easybuild/scripts/EasyBuild-reinstall-dev.sh b/easybuild/scripts/EasyBuild-reinstall-dev.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/createSubmoduleDeps.sh b/easybuild/scripts/createSubmoduleDeps.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/findUpdatedEcs.sh b/easybuild/scripts/findUpdatedEcs.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/fix_docs.py b/easybuild/scripts/fix_docs.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/install-EasyBuild-develop.sh b/easybuild/scripts/install-EasyBuild-develop.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/install-EasyBuild-sprint.sh b/easybuild/scripts/install-EasyBuild-sprint.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/install_eb_dep.sh b/easybuild/scripts/install_eb_dep.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py old mode 100644 new mode 100755 diff --git a/eb b/eb old mode 100644 new mode 100755 diff --git a/test/framework/suite.py b/test/framework/suite.py old mode 100644 new mode 100755 From 852e4c902d91bb8c442abc4cf145fc91912efbc9 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 8 Aug 2023 12:15:00 +0100 Subject: [PATCH 025/312] Updated unit tests expected output with new hooks --- test/framework/options.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index 106f951374..8d67dda656 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -717,6 +717,9 @@ def test_avail_hooks(self): " pre_testcases_hook", " post_testcases_hook", " end_hook", + " fail_hook", + " cancel_hook" + " crash_hook" '', ]) self.assertEqual(stdout, expected) From 3d61e4aadfe18faa815cd83be7f86be78cdbe48d Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 8 Aug 2023 12:22:43 +0100 Subject: [PATCH 026/312] Insert missing commas --- test/framework/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 8d67dda656..0aebead19d 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -718,8 +718,8 @@ def test_avail_hooks(self): " post_testcases_hook", " end_hook", " fail_hook", - " cancel_hook" - " crash_hook" + " cancel_hook", + " crash_hook", '', ]) self.assertEqual(stdout, expected) From b8808564b649b56270b1187a8cd09f3f54945a0a Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 8 Aug 2023 13:35:09 +0100 Subject: [PATCH 027/312] Replace spaces with tabs to match expected options output --- test/framework/options.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 0aebead19d..1d7d2c3471 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -717,9 +717,9 @@ def test_avail_hooks(self): " pre_testcases_hook", " post_testcases_hook", " end_hook", - " fail_hook", - " cancel_hook", - " crash_hook", + " fail_hook", + " cancel_hook", + " crash_hook", '', ]) self.assertEqual(stdout, expected) From b5ec611bd4a4daae9bc098eb861cb4b0d42fe2a9 Mon Sep 17 00:00:00 2001 From: XavierCS-dev <64903011+XavierCS-dev@users.noreply.github.com> Date: Tue, 8 Aug 2023 16:08:33 +0100 Subject: [PATCH 028/312] Shorten long line of imports --- easybuild/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 37c2c31318..f0932e9916 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -69,7 +69,8 @@ from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, merge_pr, new_branch_github, new_pr from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr -from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, CRASH, START, END, CANCEL, FAIL, load_hooks, run_hook +from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, CRASH, START, \ + END, CANCEL, FAIL, load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import opts_dict_to_eb_opts, parse_options, set_up_configuration, use_color from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm From 95c788f3e7f239e5c18a7d24cc4a3a4f450c0c06 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 9 Aug 2023 09:28:15 +0200 Subject: [PATCH 029/312] Update checksum type error message --- easybuild/tools/filetools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index f3bc626b2b..e7034eca5c 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1299,7 +1299,8 @@ def verify_checksum(path, checksums): # no matching checksums return False else: - raise EasyBuildError("Invalid checksum spec '%s', should be a string (MD5) or 2-tuple (type, value).", + raise EasyBuildError("Invalid checksum spec '%s', should be a string (MD5 or SHA256) " + "2-tuple (type, value) or tuple of alternative checksum specs.", checksum) actual_checksum = compute_checksum(path, typ) From f296dc5d937fd9280fc7570a0aa1679669cea532 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 9 Aug 2023 11:44:58 +0100 Subject: [PATCH 030/312] formatting of imports --- easybuild/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index f0932e9916..aaf7f71274 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -69,8 +69,8 @@ from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, merge_pr, new_branch_github, new_pr from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr -from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, CRASH, START, \ - END, CANCEL, FAIL, load_hooks, run_hook +from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, CRASH, START, END, CANCEL, FAIL +from easybuild.tools.hooks import load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import opts_dict_to_eb_opts, parse_options, set_up_configuration, use_color from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm From 0143b5cd0aa44d2b7ee9a48b0d39c1cba96dfeef Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 9 Aug 2023 16:39:03 +0100 Subject: [PATCH 031/312] enhance `get_flag` to handle lists --- easybuild/tools/toolchain/toolchain.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 308678a555..38d2f139d3 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1129,8 +1129,14 @@ def _setenv_variables(self, donotset=None, verbose=True): setvar("EBVAR%s" % key, val, verbose=False) def get_flag(self, name): - """Get compiler flag for a certain option.""" - return "-%s" % self.options.option(name) + """Get compiler flag(s) for a certain option.""" + if isinstance(self.options.option(name), str): + return "-%s" % self.options.option(name) + elif isinstance(self.options.option(name), list): + return " ".join("-%s" % x for x in self.options.option(name)) + else: + msg = "Do not know how to convert toolchain flag %s, of type %s, for use" + raise EasyBuildError(msg % (name, type(self.options.option(name)))) def toolchain_family(self): """Return toolchain family for this toolchain.""" From f2ec534da2a3caf5b98fe5a2dbd2f00cb50e8e7e Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 9 Aug 2023 16:59:18 +0100 Subject: [PATCH 032/312] maintain existing behaviour unless it is a list --- easybuild/tools/toolchain/toolchain.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 38d2f139d3..4ada51995b 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1130,13 +1130,10 @@ def _setenv_variables(self, donotset=None, verbose=True): def get_flag(self, name): """Get compiler flag(s) for a certain option.""" - if isinstance(self.options.option(name), str): - return "-%s" % self.options.option(name) - elif isinstance(self.options.option(name), list): + if isinstance(self.options.option(name), list): return " ".join("-%s" % x for x in self.options.option(name)) else: - msg = "Do not know how to convert toolchain flag %s, of type %s, for use" - raise EasyBuildError(msg % (name, type(self.options.option(name)))) + return "-%s" % self.options.option(name) def toolchain_family(self): """Return toolchain family for this toolchain.""" From 3052696ca49b3ed062d68e3e900758b8b3bbe454 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Wed, 9 Aug 2023 17:31:20 +0100 Subject: [PATCH 033/312] also accept tuple --- easybuild/tools/toolchain/toolchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 4ada51995b..b9d489852e 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1130,7 +1130,7 @@ def _setenv_variables(self, donotset=None, verbose=True): def get_flag(self, name): """Get compiler flag(s) for a certain option.""" - if isinstance(self.options.option(name), list): + if isinstance(self.options.option(name), list) or isinstance(self.options.option(name), tuple): return " ".join("-%s" % x for x in self.options.option(name)) else: return "-%s" % self.options.option(name) From 2646f7584a1d0253a9afea4cdd0f766f6f53e26d Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Wed, 9 Aug 2023 18:24:58 +0100 Subject: [PATCH 034/312] add tests --- easybuild/tools/toolchain/toolchain.py | 4 ++-- test/framework/toolchain.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index b9d489852e..a16fe35dc0 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1130,8 +1130,8 @@ def _setenv_variables(self, donotset=None, verbose=True): def get_flag(self, name): """Get compiler flag(s) for a certain option.""" - if isinstance(self.options.option(name), list) or isinstance(self.options.option(name), tuple): - return " ".join("-%s" % x for x in self.options.option(name)) + if isinstance(self.options.option(name), list): + return " ".join("-%s" % x for x in list(self.options.option(name))) else: return "-%s" % self.options.option(name) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 1c36431c45..cf0369764c 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -2950,6 +2950,20 @@ def test_env_vars_external_module(self): expected = {} self.assertEqual(res, expected) + def test_get_flag(self): + """Test get_flag function""" + tc = self.get_toolchain('gompi', version='2018a') + + checks = { + '-a': 'a', + '-openmp': 'openmp', + '-foo': ['foo'], + '-foo -bar': ['foo', 'bar'], + } + + for flagstring, flags in checks.items(): + tc.options.options_map['openmp'] = flags + self.assertEqual(tc.get_flag('openmp'), flagstring) def suite(): """ return all the tests""" From 587f64d893529c1b12b65de8e45a2741b5358f8c Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Wed, 9 Aug 2023 18:27:03 +0100 Subject: [PATCH 035/312] f8 --- test/framework/toolchain.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index cf0369764c..d80d41c788 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -2965,6 +2965,7 @@ def test_get_flag(self): tc.options.options_map['openmp'] = flags self.assertEqual(tc.get_flag('openmp'), flagstring) + def suite(): """ return all the tests""" return TestLoaderFiltered().loadTestsFromTestCase(ToolchainTest, sys.argv[1:]) From 0e751c9319be06b1c1e60f284510e37961159059 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 10:33:12 +0100 Subject: [PATCH 036/312] Reorder failure hooks to be in alphabetical order --- easybuild/main.py | 3 ++- easybuild/tools/hooks.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index b5c2f9a01e..621871cbb0 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -69,7 +69,8 @@ from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, merge_pr, new_branch_github, new_pr from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr -from easybuild.tools.hooks import CRASH, START, END, CANCEL, FAIL, load_hooks, run_hook +from easybuild.tools.hooks import CRASH, START, \ + END, CANCEL, FAIL, load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import opts_dict_to_eb_opts, parse_options, set_up_configuration, use_color from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index de07866c16..8e8eb2302c 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -65,9 +65,9 @@ MODULE_WRITE = 'module_write' END = 'end' -FAIL = 'fail' CANCEL = 'cancel' CRASH = 'crash' +FAIL = 'fail' PRE_PREF = 'pre_' POST_PREF = 'post_' @@ -102,9 +102,9 @@ ] + [p + x for x in STEP_NAMES[STEP_NAMES.index(MODULE_STEP)+1:] for p in [PRE_PREF, POST_PREF]] + [ END, - FAIL, CANCEL, CRASH, + FAIL, ] KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES] From 8a474bb8ad69d756c7c606fcd3dc80c76a382c16 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 10:55:32 +0100 Subject: [PATCH 037/312] Fix output tests for hook order --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index d889fc6b6c..5b1ef19825 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -719,9 +719,9 @@ def test_avail_hooks(self): " post_testcases_hook", " post_build_and_install_loop_hook", " end_hook", - " fail_hook", " cancel_hook", " crash_hook", + " fail_hook", '', ]) self.assertEqual(stdout, expected) From ec87c8608cc4748aaa6367b23bfed8403f545cd2 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 14:35:42 +0100 Subject: [PATCH 038/312] Alter refactor to define variables --- easybuild/main.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index aaf7f71274..a05fae4604 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -580,7 +580,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session return overall_success -def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): +def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, testing=False, modtool=None): """ Main function: parse command line options, and act accordingly. :param args: command line arguments to use @@ -600,6 +600,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # So emulate this here to allow (module) scripts depending on that to work if '_' not in os.environ: os.environ['_'] = sys.executable + + if any([args, logfile, testing]): + eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() @@ -732,11 +735,16 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): cleanup(logfile, eb_tmpdir, testing, silent=False) +def prepare_main(args=None, logfile=None, testing=None): + eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) + return eb_go, cfg_settings + + if __name__ == "__main__": - eb_go = parse_options() + eb_go, cfg_settings = prepare_main() hooks = load_hooks(eb_go.options.hooks) try: - main() + main(eb_go, cfg_settings) except EasyBuildError as err: run_hook(FAIL, hooks, args=[err]) print_error(err.msg) From 2e8b80f8986397281fa7a788b5b564fae47fe8ac Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 14:37:22 +0100 Subject: [PATCH 039/312] Comment out line overriding variables defined elsewhere --- easybuild/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index a05fae4604..ebcc2bd917 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -606,7 +606,7 @@ def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() - eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) + # eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) options, orig_paths = eb_go.options, eb_go.args if 'python2' not in build_option('silence_deprecation_warnings'): From 81e534420a365177a7cbb161b8746dfc1548a987 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 14:47:52 +0100 Subject: [PATCH 040/312] Fix style issues --- easybuild/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index ebcc2bd917..f376f0d43f 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -72,7 +72,7 @@ from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, CRASH, START, END, CANCEL, FAIL from easybuild.tools.hooks import load_hooks, run_hook from easybuild.tools.modules import modules_tool -from easybuild.tools.options import opts_dict_to_eb_opts, parse_options, set_up_configuration, use_color +from easybuild.tools.options import opts_dict_to_eb_opts, set_up_configuration, use_color from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs @@ -600,7 +600,7 @@ def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, # So emulate this here to allow (module) scripts depending on that to work if '_' not in os.environ: os.environ['_'] = sys.executable - + if any([args, logfile, testing]): eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) From 0ac9daf0c979ce5d15bee18f53fde3ad9560f1b3 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 15:42:50 +0100 Subject: [PATCH 041/312] Update main docstring and add prepare_main docstring --- easybuild/main.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/easybuild/main.py b/easybuild/main.py index f376f0d43f..123a0e965a 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -583,6 +583,8 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, testing=False, modtool=None): """ Main function: parse command line options, and act accordingly. + :param: eb_go: easybuild general options object + :param: cfg_settings: configuration settings of easybuild :param args: command line arguments to use :param logfile: log file to use :param do_build: whether or not to actually perform the build @@ -736,6 +738,13 @@ def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, def prepare_main(args=None, logfile=None, testing=None): + """ + Prepare Main: Set up eb_go and consequently, cfg_settings in order to obtain the hooks, + and prevent the configuration being set up twice. + :param args: command line arguments to take into account when parsing the EasyBuild configuration settings + :param logfile: log file to use + :param testing: enable testing mode + """ eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) return eb_go, cfg_settings From 22b44297912d32824c395b41c7ab247d781c11d1 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 16:08:57 +0100 Subject: [PATCH 042/312] Remove uncessary comment --- easybuild/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 123a0e965a..86360006d1 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -608,7 +608,6 @@ def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() - # eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) options, orig_paths = eb_go.options, eb_go.args if 'python2' not in build_option('silence_deprecation_warnings'): From 1751fe309e81e5e7e1d1d4b9504424d97575c15d Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 16:24:17 +0100 Subject: [PATCH 043/312] Only set eb_go and cfg_settings if not already --- easybuild/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 86360006d1..78f4eb7343 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -603,7 +603,7 @@ def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, if '_' not in os.environ: os.environ['_'] = sys.executable - if any([args, logfile, testing]): + if not all([eb_go, cfg_settings]): eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) # purposely session state very early, to avoid modules loaded by EasyBuild meddling in From c4be4ca3d9e54622502ccccc05c0fb83810f8e95 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 16:25:29 +0100 Subject: [PATCH 044/312] Move setup config to occure before os.environ is checked --- easybuild/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 78f4eb7343..848c7f1baf 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -593,6 +593,9 @@ def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, register_lock_cleanup_signal_handlers() + if not all([eb_go, cfg_settings]): + eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) + # if $CDPATH is set, unset it, it'll only cause trouble... # see https://github.com/easybuilders/easybuild-framework/issues/2944 if 'CDPATH' in os.environ: @@ -603,9 +606,6 @@ def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, if '_' not in os.environ: os.environ['_'] = sys.executable - if not all([eb_go, cfg_settings]): - eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) - # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() options, orig_paths = eb_go.options, eb_go.args From 58623de7eebe1a4215d588978e617b9974e674be Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 16:26:55 +0100 Subject: [PATCH 045/312] Change pprepare_main description --- easybuild/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 848c7f1baf..ef69ca19d3 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -738,8 +738,7 @@ def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, def prepare_main(args=None, logfile=None, testing=None): """ - Prepare Main: Set up eb_go and consequently, cfg_settings in order to obtain the hooks, - and prevent the configuration being set up twice. + Prepare for calling main function by setting up the EasyBuild configuration. :param args: command line arguments to take into account when parsing the EasyBuild configuration settings :param logfile: log file to use :param testing: enable testing mode From 8148fb5f0a4af7d253ce30a7ac3fdf03161e0d40 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 16:28:58 +0100 Subject: [PATCH 046/312] Add return to prepare_main docstring --- easybuild/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index ef69ca19d3..1a22daf318 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -738,10 +738,11 @@ def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, def prepare_main(args=None, logfile=None, testing=None): """ - Prepare for calling main function by setting up the EasyBuild configuration. + Prepare for calling main function by setting up the EasyBuild configuration :param args: command line arguments to take into account when parsing the EasyBuild configuration settings :param logfile: log file to use :param testing: enable testing mode + return: easybuild options and configuration settings """ eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) return eb_go, cfg_settings From f2f6b45230f64dc18eef80a695e69fb0ee0f468b Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 16:31:07 +0100 Subject: [PATCH 047/312] Move eb_go and cfg_settings to end of parameters to prevent breaking function calls --- easybuild/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 1a22daf318..6413b42c6c 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -580,7 +580,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session return overall_success -def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, testing=False, modtool=None): +def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, eb_go=None, cfg_settings=None): """ Main function: parse command line options, and act accordingly. :param: eb_go: easybuild general options object @@ -742,7 +742,7 @@ def prepare_main(args=None, logfile=None, testing=None): :param args: command line arguments to take into account when parsing the EasyBuild configuration settings :param logfile: log file to use :param testing: enable testing mode - return: easybuild options and configuration settings + :return: easybuild options and configuration settings """ eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) return eb_go, cfg_settings @@ -752,7 +752,7 @@ def prepare_main(args=None, logfile=None, testing=None): eb_go, cfg_settings = prepare_main() hooks = load_hooks(eb_go.options.hooks) try: - main(eb_go, cfg_settings) + main(eb_go=eb_go, cfg_settings=cfg_settings) except EasyBuildError as err: run_hook(FAIL, hooks, args=[err]) print_error(err.msg) From 56def6bf5f740fe754f742aa712e6adcd0d7f79e Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 16:46:12 +0100 Subject: [PATCH 048/312] Add init_session_state and hooks to prepare_main returns --- easybuild/main.py | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 6413b42c6c..500ffabfe9 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -580,7 +580,8 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session return overall_success -def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, eb_go=None, cfg_settings=None): +def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, eb_go=None, cfg_settings=None, + init_session_state=None, hooks=None): """ Main function: parse command line options, and act accordingly. :param: eb_go: easybuild general options object @@ -591,23 +592,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, eb :param testing: enable testing mode """ - register_lock_cleanup_signal_handlers() - - if not all([eb_go, cfg_settings]): - eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) - - # if $CDPATH is set, unset it, it'll only cause trouble... - # see https://github.com/easybuilders/easybuild-framework/issues/2944 - if 'CDPATH' in os.environ: - del os.environ['CDPATH'] + if not all([eb_go, cfg_settings, init_session_state, hooks]): + eb_go, cfg_settings, init_session_state, hooks = prepare_main(args=args, logfile=logfile, testing=testing) - # When EB is run via `exec` the special bash variable $_ is not set - # So emulate this here to allow (module) scripts depending on that to work - if '_' not in os.environ: - os.environ['_'] = sys.executable - - # purposely session state very early, to avoid modules loaded by EasyBuild meddling in - init_session_state = session_state() options, orig_paths = eb_go.options, eb_go.args if 'python2' not in build_option('silence_deprecation_warnings'): @@ -742,17 +729,31 @@ def prepare_main(args=None, logfile=None, testing=None): :param args: command line arguments to take into account when parsing the EasyBuild configuration settings :param logfile: log file to use :param testing: enable testing mode - :return: easybuild options and configuration settings + :return: easybuild options, configuration settings and session state """ + register_lock_cleanup_signal_handlers() + + # if $CDPATH is set, unset it, it'll only cause trouble... + # see https://github.com/easybuilders/easybuild-framework/issues/2944 + if 'CDPATH' in os.environ: + del os.environ['CDPATH'] + + # When EB is run via `exec` the special bash variable $_ is not set + # So emulate this here to allow (module) scripts depending on that to work + if '_' not in os.environ: + os.environ['_'] = sys.executable + + # purposely session state very early, to avoid modules loaded by EasyBuild meddling in + init_session_state = session_state() eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) - return eb_go, cfg_settings + hooks = load_hooks(eb_go.options.hooks) + return eb_go, cfg_settings, init_session_state, hooks if __name__ == "__main__": - eb_go, cfg_settings = prepare_main() - hooks = load_hooks(eb_go.options.hooks) + eb_go, cfg_settings, init_session_state, hooks = prepare_main() try: - main(eb_go=eb_go, cfg_settings=cfg_settings) + main(eb_go=eb_go, cfg_settings=cfg_settings, init_session_state=init_session_state, hooks=hooks) except EasyBuildError as err: run_hook(FAIL, hooks, args=[err]) print_error(err.msg) From d8a34e4a898c6a3e72f0760a6e26a3c0e6a751d6 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 16:47:57 +0100 Subject: [PATCH 049/312] Update docstrings to match new parameters and return values --- easybuild/main.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 500ffabfe9..1f71c60ba3 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -584,12 +584,14 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, eb init_session_state=None, hooks=None): """ Main function: parse command line options, and act accordingly. - :param: eb_go: easybuild general options object - :param: cfg_settings: configuration settings of easybuild :param args: command line arguments to use :param logfile: log file to use :param do_build: whether or not to actually perform the build :param testing: enable testing mode + :param eb_go: easybuild general options object + :param cfg_settings: configuration settings of easybuild + :param init_session_state: session state + :param hooks: available hooks """ if not all([eb_go, cfg_settings, init_session_state, hooks]): @@ -729,7 +731,7 @@ def prepare_main(args=None, logfile=None, testing=None): :param args: command line arguments to take into account when parsing the EasyBuild configuration settings :param logfile: log file to use :param testing: enable testing mode - :return: easybuild options, configuration settings and session state + :return: easybuild options, configuration settings, session state and hooks """ register_lock_cleanup_signal_handlers() From fcc6fee8a246dd4c4c2d004c65d7372a718580d8 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 16:57:36 +0100 Subject: [PATCH 050/312] Add or condition to ensure test passes --- easybuild/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 1f71c60ba3..82c5fbac3e 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -594,7 +594,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, eb :param hooks: available hooks """ - if not all([eb_go, cfg_settings, init_session_state, hooks]): + if not all([eb_go, cfg_settings, init_session_state, hooks]) or any([args, logfile, testing]): eb_go, cfg_settings, init_session_state, hooks = prepare_main(args=args, logfile=logfile, testing=testing) options, orig_paths = eb_go.options, eb_go.args From 4146b46ef0ff78920169255a8d93d5bc1754d08b Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Fri, 11 Aug 2023 09:50:00 +0100 Subject: [PATCH 051/312] Separate check on hooks to be None --- easybuild/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 82c5fbac3e..2a3dbe5781 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -594,7 +594,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, eb :param hooks: available hooks """ - if not all([eb_go, cfg_settings, init_session_state, hooks]) or any([args, logfile, testing]): + if not all([eb_go, cfg_settings, init_session_state, hooks]) or hooks is None or any([args, logfile, testing]): eb_go, cfg_settings, init_session_state, hooks = prepare_main(args=args, logfile=logfile, testing=testing) options, orig_paths = eb_go.options, eb_go.args From d9b15f04dcc5abc5fa93baa654a9a5d6965ccead Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Fri, 11 Aug 2023 10:01:36 +0100 Subject: [PATCH 052/312] Improve clarity when checking for None values in main --- easybuild/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 2a3dbe5781..8b4929d035 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -594,7 +594,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, eb :param hooks: available hooks """ - if not all([eb_go, cfg_settings, init_session_state, hooks]) or hooks is None or any([args, logfile, testing]): + if any(i is None for i in [eb_go, cfg_settings, init_session_state, hooks]) or any([args, logfile, testing]): eb_go, cfg_settings, init_session_state, hooks = prepare_main(args=args, logfile=logfile, testing=testing) options, orig_paths = eb_go.options, eb_go.args From 7d31bb8e2b209a77ecddbf966c7f3a7bdf1dfeb6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 11 Aug 2023 12:06:47 +0200 Subject: [PATCH 053/312] clean up passing of data produced by prepare_main into main --- easybuild/main.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 8b4929d035..f20f5b4152 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -580,22 +580,22 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session return overall_success -def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, eb_go=None, cfg_settings=None, - init_session_state=None, hooks=None): +def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, prepared_cfg_data=None): """ Main function: parse command line options, and act accordingly. :param args: command line arguments to use :param logfile: log file to use :param do_build: whether or not to actually perform the build :param testing: enable testing mode + :param prepared_cfg_data: prepared configuration data for main function, as returned by prepare_main (or None) :param eb_go: easybuild general options object :param cfg_settings: configuration settings of easybuild :param init_session_state: session state - :param hooks: available hooks """ - - if any(i is None for i in [eb_go, cfg_settings, init_session_state, hooks]) or any([args, logfile, testing]): - eb_go, cfg_settings, init_session_state, hooks = prepare_main(args=args, logfile=logfile, testing=testing) + if prepared_cfg_data is None or any([args, logfile, testing]): + init_session_state, eb_go, cfg_settings = prepare_main(args=args, logfile=logfile, testing=testing) + else: + init_session_state, eb_go, cfg_settings = prepared_cfg_data options, orig_paths = eb_go.options, eb_go.args @@ -731,7 +731,7 @@ def prepare_main(args=None, logfile=None, testing=None): :param args: command line arguments to take into account when parsing the EasyBuild configuration settings :param logfile: log file to use :param testing: enable testing mode - :return: easybuild options, configuration settings, session state and hooks + :return: 3-tuple with initial session state data, EasyBuildOptions instance, and tuple with configuration settings """ register_lock_cleanup_signal_handlers() @@ -748,14 +748,15 @@ def prepare_main(args=None, logfile=None, testing=None): # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) - hooks = load_hooks(eb_go.options.hooks) - return eb_go, cfg_settings, init_session_state, hooks + + return init_session_state, eb_go, cfg_settings if __name__ == "__main__": - eb_go, cfg_settings, init_session_state, hooks = prepare_main() + init_session_state, eb_go, cfg_settings = prepare_main() + hooks = load_hooks(eb_go.options.hooks) try: - main(eb_go=eb_go, cfg_settings=cfg_settings, init_session_state=init_session_state, hooks=hooks) + main(prepared_cfg_data=(init_session_state, eb_go, cfg_settings)) except EasyBuildError as err: run_hook(FAIL, hooks, args=[err]) print_error(err.msg) From 20c99cdaf00878bb4583396c930a049bd6ee8a20 Mon Sep 17 00:00:00 2001 From: XavierCS-dev <64903011+XavierCS-dev@users.noreply.github.com> Date: Fri, 11 Aug 2023 11:15:21 +0100 Subject: [PATCH 054/312] Remove unnecessary parameter descriptions Co-authored-by: Simon Branford <4967+branfosj@users.noreply.github.com> --- easybuild/main.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index f20f5b4152..24f994da14 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -588,9 +588,6 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr :param do_build: whether or not to actually perform the build :param testing: enable testing mode :param prepared_cfg_data: prepared configuration data for main function, as returned by prepare_main (or None) - :param eb_go: easybuild general options object - :param cfg_settings: configuration settings of easybuild - :param init_session_state: session state """ if prepared_cfg_data is None or any([args, logfile, testing]): init_session_state, eb_go, cfg_settings = prepare_main(args=args, logfile=logfile, testing=testing) From ddf0c947cba3fe05e1172c7faee98d56b581ee70 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 11 Aug 2023 19:02:53 +0200 Subject: [PATCH 055/312] trigger hook before/after running a shell command (disabled in tools.systemtools for functions that are called before configuration is fully in place so build options are defined) --- easybuild/tools/hooks.py | 4 ++++ easybuild/tools/options.py | 2 +- easybuild/tools/run.py | 12 +++++++++++- easybuild/tools/systemtools.py | 24 +++++++++++++----------- test/framework/options.py | 2 ++ 5 files changed, 31 insertions(+), 13 deletions(-) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 7a104d6277..a3586cfb4b 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -70,6 +70,8 @@ CRASH = 'crash' FAIL = 'fail' +RUN_SHELL_CMD = 'run_shell_cmd' + PRE_PREF = 'pre_' POST_PREF = 'post_' HOOK_SUFF = '_hook' @@ -108,6 +110,8 @@ CANCEL, CRASH, FAIL, + PRE_PREF + RUN_SHELL_CMD, + POST_PREF + RUN_SHELL_CMD, ] KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES] diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 761cdc262f..5acf1780a2 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1894,7 +1894,7 @@ def set_tmpdir(tmpdir=None, raise_error=False): os.close(fd) os.chmod(tmptest_file, 0o700) if not run_cmd(tmptest_file, simple=True, log_ok=False, regexp=False, force_in_dry_run=True, trace=False, - stream_output=False): + stream_output=False, with_hooks=False): msg = "The temporary directory (%s) does not allow to execute files. " % tempfile.gettempdir() msg += "This can cause problems in the build process, consider using --tmpdir." if raise_error: diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 1b8c385563..8f2ba68c5d 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -50,6 +50,7 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, time_str_since from easybuild.tools.config import ERROR, IGNORE, WARN, build_option +from easybuild.tools.hooks import RUN_SHELL_CMD, load_hooks, run_hook from easybuild.tools.py2vs3 import string_type from easybuild.tools.utilities import trace_msg @@ -131,7 +132,8 @@ def get_output_from_process(proc, read_size=None, asynchronous=False): @run_cmd_cache def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None, - force_in_dry_run=False, verbose=True, shell=None, trace=True, stream_output=None, asynchronous=False): + force_in_dry_run=False, verbose=True, shell=None, trace=True, stream_output=None, asynchronous=False, + with_hooks=True): """ Run specified command (in a subshell) :param cmd: command to run @@ -148,6 +150,7 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True :param trace: print command being executed as part of trace output :param stream_output: enable streaming command output to stdout :param asynchronous: run command asynchronously (returns subprocess.Popen instance if set to True) + :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined) """ cwd = os.getcwd() @@ -233,6 +236,10 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True else: raise EasyBuildError("Don't know how to prefix with /usr/bin/env for commands of type %s", type(cmd)) + if with_hooks: + hooks = load_hooks(build_option('hooks')) + run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd]) + _log.info('running cmd: %s ' % cmd) try: proc = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -240,6 +247,9 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True except OSError as err: raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err) + if with_hooks: + run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd]) + if inp: proc.stdin.write(inp.encode()) proc.stdin.close() diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index cab4b00055..b81e018bba 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -274,7 +274,7 @@ def get_avail_core_count(): core_cnt = int(sum(sched_getaffinity())) else: # BSD-type systems - out, _ = run_cmd('sysctl -n hw.ncpu', force_in_dry_run=True, trace=False, stream_output=False) + out, _ = run_cmd('sysctl -n hw.ncpu', force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False) try: if int(out) > 0: core_cnt = int(out) @@ -311,7 +311,7 @@ def get_total_memory(): elif os_type == DARWIN: cmd = "sysctl -n hw.memsize" _log.debug("Trying to determine total memory size on Darwin via cmd '%s'", cmd) - out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False) if ec == 0: memtotal = int(out.strip()) // (1024**2) @@ -393,14 +393,15 @@ def get_cpu_vendor(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.vendor" - out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False, with_hooks=False) out = out.strip() if ec == 0 and out in VENDOR_IDS: vendor = VENDOR_IDS[out] _log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd)) else: cmd = "sysctl -n machdep.cpu.brand_string" - out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False, + with_hooks=False) out = out.strip().split(' ')[0] if ec == 0 and out in CPU_VENDORS: vendor = out @@ -503,7 +504,7 @@ def get_cpu_model(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.brand_string" - out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False) if ec == 0: model = out.strip() _log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model)) @@ -548,7 +549,7 @@ def get_cpu_speed(): elif os_type == DARWIN: cmd = "sysctl -n hw.cpufrequency_max" _log.debug("Trying to determine CPU frequency on Darwin via cmd '%s'" % cmd) - out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False) out = out.strip() cpu_freq = None if ec == 0 and out: @@ -596,7 +597,8 @@ def get_cpu_features(): for feature_set in ['extfeatures', 'features', 'leaf7_features']: cmd = "sysctl -n machdep.cpu.%s" % feature_set _log.debug("Trying to determine CPU features on Darwin via cmd '%s'", cmd) - out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False, + with_hooks=False) if ec == 0: cpu_feat.extend(out.strip().lower().split()) @@ -624,7 +626,7 @@ def get_gpu_info(): cmd = "nvidia-smi --query-gpu=gpu_name,driver_version --format=csv,noheader" _log.debug("Trying to determine NVIDIA GPU info on Linux via cmd '%s'", cmd) out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, - force_in_dry_run=True, trace=False, stream_output=False) + force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False) if ec == 0: for line in out.strip().split('\n'): nvidia_gpu_info = gpu_info.setdefault('NVIDIA', {}) @@ -643,14 +645,14 @@ def get_gpu_info(): cmd = "rocm-smi --showdriverversion --csv" _log.debug("Trying to determine AMD GPU driver on Linux via cmd '%s'", cmd) out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, - force_in_dry_run=True, trace=False, stream_output=False) + force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False) if ec == 0: amd_driver = out.strip().split('\n')[1].split(',')[1] cmd = "rocm-smi --showproductname --csv" _log.debug("Trying to determine AMD GPU info on Linux via cmd '%s'", cmd) out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, - force_in_dry_run=True, trace=False, stream_output=False) + force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False) if ec == 0: for line in out.strip().split('\n')[1:]: amd_card_series = line.split(',')[1] @@ -898,7 +900,7 @@ def get_tool_version(tool, version_option='--version', ignore_ec=False): Output is returned as a single-line string (newlines are replaced by '; '). """ out, ec = run_cmd(' '.join([tool, version_option]), simple=False, log_ok=False, force_in_dry_run=True, - trace=False, stream_output=False) + trace=False, stream_output=False, with_hooks=False) if not ignore_ec and ec: _log.warning("Failed to determine version of %s using '%s %s': %s" % (tool, tool, version_option, out)) return UNKNOWN diff --git a/test/framework/options.py b/test/framework/options.py index 2ef6488d16..78c3a495e6 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -724,6 +724,8 @@ def test_avail_hooks(self): " cancel_hook", " crash_hook", " fail_hook", + " pre_run_shell_cmd_hook", + " post_run_shell_cmd_hook", '', ]) self.assertEqual(stdout, expected) From 1b798e1dacc78925f8eda8725a198071a142926a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 11 Aug 2023 19:26:29 +0200 Subject: [PATCH 056/312] extend hook tests to also check use of run_shell_cmd, fail, and build_and_install_loop hooks --- test/framework/hooks.py | 42 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/test/framework/hooks.py b/test/framework/hooks.py index fad251b040..0743cc16b9 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -52,6 +52,9 @@ def setUp(self): 'def parse_hook(ec):', ' print("Parse hook with argument %s" % ec)', '', + 'def pre_build_and_install_loop_hook(ecs):', + ' print("About to start looping for %d easyconfigs!" % len(ecs))', + '', 'def foo():', ' print("running foo helper method")', '', @@ -64,6 +67,12 @@ def setUp(self): '', 'def pre_single_extension_hook(ext):', ' print("this is run before installing an extension")', + '', + 'def pre_run_shell_cmd_hook(cmd):', + ' print("this is run before running command \'%s\'" % cmd)', + '', + 'def fail_hook(err):', + ' print("EasyBuild FAIL: %s" % err)', ]) write_file(self.test_hooks_pymod, test_hooks_pymod_txt) @@ -74,11 +83,14 @@ def test_load_hooks(self): hooks = load_hooks(self.test_hooks_pymod) - self.assertEqual(len(hooks), 5) + self.assertEqual(len(hooks), 8) expected = [ + 'fail_hook', 'parse_hook', 'post_configure_hook', + 'pre_build_and_install_loop_hook', 'pre_install_hook', + 'pre_run_shell_cmd_hook', 'pre_single_extension_hook', 'start_hook', ] @@ -113,6 +125,9 @@ def test_find_hook(self): pre_install_hook = [hooks[k] for k in hooks if k == 'pre_install_hook'][0] pre_single_extension_hook = [hooks[k] for k in hooks if k == 'pre_single_extension_hook'][0] start_hook = [hooks[k] for k in hooks if k == 'start_hook'][0] + pre_run_shell_cmd_hook = [hooks[k] for k in hooks if k == 'pre_run_shell_cmd_hook'][0] + fail_hook = [hooks[k] for k in hooks if k == 'fail_hook'][0] + pre_build_and_install_loop_hook = [hooks[k] for k in hooks if k == 'pre_build_and_install_loop_hook'][0] self.assertEqual(find_hook('configure', hooks), None) self.assertEqual(find_hook('configure', hooks, pre_step_hook=True), None) @@ -138,6 +153,19 @@ def test_find_hook(self): self.assertEqual(find_hook('start', hooks, pre_step_hook=True), None) self.assertEqual(find_hook('start', hooks, post_step_hook=True), None) + self.assertEqual(find_hook('run_shell_cmd', hooks), None) + self.assertEqual(find_hook('run_shell_cmd', hooks, pre_step_hook=True), pre_run_shell_cmd_hook) + self.assertEqual(find_hook('run_shell_cmd', hooks, post_step_hook=True), None) + + self.assertEqual(find_hook('fail', hooks), fail_hook) + self.assertEqual(find_hook('fail', hooks, pre_step_hook=True), None) + self.assertEqual(find_hook('fail', hooks, post_step_hook=True), None) + + hook_name = 'build_and_install_loop' + self.assertEqual(find_hook(hook_name, hooks), None) + self.assertEqual(find_hook(hook_name, hooks, pre_step_hook=True), pre_build_and_install_loop_hook) + self.assertEqual(find_hook(hook_name, hooks, post_step_hook=True), None) + def test_run_hook(self): """Test for run_hook function.""" @@ -149,17 +177,21 @@ def test_run_hook(self): self.mock_stderr(True) run_hook('start', hooks) run_hook('parse', hooks, args=[''], msg="Running parse hook for example.eb...") + run_hook('build_and_install_loop', hooks, args=[['ec1', 'ec2']], pre_step_hook=True) run_hook('configure', hooks, pre_step_hook=True, args=[None]) run_hook('configure', hooks, post_step_hook=True, args=[None]) run_hook('build', hooks, pre_step_hook=True, args=[None]) + run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make -j 3"]) run_hook('build', hooks, post_step_hook=True, args=[None]) run_hook('install', hooks, pre_step_hook=True, args=[None]) + run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make install"]) run_hook('install', hooks, post_step_hook=True, args=[None]) run_hook('extensions', hooks, pre_step_hook=True, args=[None]) for _ in range(3): run_hook('single_extension', hooks, pre_step_hook=True, args=[None]) run_hook('single_extension', hooks, post_step_hook=True, args=[None]) run_hook('extensions', hooks, post_step_hook=True, args=[None]) + run_hook('fail', hooks, args=[EasyBuildError('oops')]) stdout = self.get_stdout() stderr = self.get_stderr() self.mock_stdout(False) @@ -170,17 +202,25 @@ def test_run_hook(self): "this is triggered at the very beginning", "== Running parse hook for example.eb...", "Parse hook with argument ", + "== Running pre-build_and_install_loop hook...", + "About to start looping for 2 easyconfigs!", "== Running post-configure hook...", "this is run after configure step", "running foo helper method", + "== Running pre-run_shell_cmd hook...", + "this is run before running command 'make -j 3'", "== Running pre-install hook...", "this is run before install step", + "== Running pre-run_shell_cmd hook...", + "this is run before running command 'make install'", "== Running pre-single_extension hook...", "this is run before installing an extension", "== Running pre-single_extension hook...", "this is run before installing an extension", "== Running pre-single_extension hook...", "this is run before installing an extension", + "== Running fail hook...", + "EasyBuild FAIL: 'oops'", ]) self.assertEqual(stdout.strip(), expected_stdout) From 7ffa35682314865f11ca63c886d9942ce6971526 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 11 Aug 2023 20:04:50 +0200 Subject: [PATCH 057/312] also run pre/post run_shell_cmd hooks when running interactive commands via run_cmd_qa --- easybuild/tools/hooks.py | 8 +++++--- easybuild/tools/run.py | 5 +++++ test/framework/hooks.py | 12 +++++++++--- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index a3586cfb4b..88fa3a85d3 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -209,7 +209,7 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False): return res -def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, msg=None): +def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, kwargs=None, msg=None): """ Run hook with specified label and return result of calling the hook or None. @@ -225,6 +225,8 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, if hook: if args is None: args = [] + if kwargs is None: + kwargs = {} if pre_step_hook: label = 'pre-' + label @@ -236,6 +238,6 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, if build_option('debug'): print_msg(msg) - _log.info("Running '%s' hook function (arguments: %s)...", hook.__name__, args) - res = hook(*args) + _log.info("Running '%s' hook function (args: %s, keyword args: %s)...", hook.__name__, args, kwargs) + res = hook(*args, **kwargs) return res diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 8f2ba68c5d..c49790a497 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -519,6 +519,9 @@ def get_proc(): if cmd_log: cmd_log.close() + hooks = load_hooks(build_option('hooks')) + run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'interactive': True}) + with get_proc() as proc: ec = proc.poll() stdout_err = '' @@ -609,6 +612,8 @@ def get_proc(): except IOError as err: _log.debug("runqanda cmd %s: remaining data read failed: %s", cmd, err) + run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs={'interactive': True}) + if trace: trace_msg("interactive command completed: exit %s, ran in %s" % (ec, time_str_since(start_time))) diff --git a/test/framework/hooks.py b/test/framework/hooks.py index 0743cc16b9..4737f1a991 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -68,8 +68,11 @@ def setUp(self): 'def pre_single_extension_hook(ext):', ' print("this is run before installing an extension")', '', - 'def pre_run_shell_cmd_hook(cmd):', - ' print("this is run before running command \'%s\'" % cmd)', + 'def pre_run_shell_cmd_hook(cmd, interactive=False):', + ' if interactive:', + ' print("this is run before running interactive command \'%s\'" % cmd)', + ' else:', + ' print("this is run before running command \'%s\'" % cmd)', '', 'def fail_hook(err):', ' print("EasyBuild FAIL: %s" % err)', @@ -179,12 +182,13 @@ def test_run_hook(self): run_hook('parse', hooks, args=[''], msg="Running parse hook for example.eb...") run_hook('build_and_install_loop', hooks, args=[['ec1', 'ec2']], pre_step_hook=True) run_hook('configure', hooks, pre_step_hook=True, args=[None]) + run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["configure.sh"], kwargs={'interactive': True}) run_hook('configure', hooks, post_step_hook=True, args=[None]) run_hook('build', hooks, pre_step_hook=True, args=[None]) run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make -j 3"]) run_hook('build', hooks, post_step_hook=True, args=[None]) run_hook('install', hooks, pre_step_hook=True, args=[None]) - run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make install"]) + run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make install"], kwargs={}) run_hook('install', hooks, post_step_hook=True, args=[None]) run_hook('extensions', hooks, pre_step_hook=True, args=[None]) for _ in range(3): @@ -204,6 +208,8 @@ def test_run_hook(self): "Parse hook with argument ", "== Running pre-build_and_install_loop hook...", "About to start looping for 2 easyconfigs!", + "== Running pre-run_shell_cmd hook...", + "this is run before running interactive command 'configure.sh'", "== Running post-configure hook...", "this is run after configure step", "running foo helper method", From 63271d430c483b48ee6540621b598bed90f6d489 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 11 Aug 2023 20:11:07 +0200 Subject: [PATCH 058/312] allow tweaking command to run via pre-run_shell_cmd hook --- easybuild/tools/run.py | 15 +++++++++++---- test/framework/hooks.py | 5 ++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index c49790a497..7d6e66ccf5 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -238,7 +238,10 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True if with_hooks: hooks = load_hooks(build_option('hooks')) - run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd]) + hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd]) + if isinstance(hook_res, string_type): + cmd, old_cmd = hook_res, cmd + self.log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd) _log.info('running cmd: %s ' % cmd) try: @@ -495,6 +498,13 @@ def check_answers_list(answers): # Part 2: Run the command and answer questions # - this needs asynchronous stdout + hooks = load_hooks(build_option('hooks')) + hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'interactive': True}) + if isinstance(hook_res, string_type): + cmd, old_cmd = hook_res, cmd + self.log.info("Interactive command to run was changed by pre-%s hook: '%s' (was: '%s')", + RUN_SHELL_CMD, cmd, old_cmd) + # # Log command output if cmd_log: cmd_log.write("# output for interactive command: %s\n\n" % cmd) @@ -519,9 +529,6 @@ def get_proc(): if cmd_log: cmd_log.close() - hooks = load_hooks(build_option('hooks')) - run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'interactive': True}) - with get_proc() as proc: ec = proc.poll() stdout_err = '' diff --git a/test/framework/hooks.py b/test/framework/hooks.py index 4737f1a991..1be55b8ac7 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -73,6 +73,8 @@ def setUp(self): ' print("this is run before running interactive command \'%s\'" % cmd)', ' else:', ' print("this is run before running command \'%s\'" % cmd)', + ' if cmd == "make install":', + ' return "sudo " + cmd', '', 'def fail_hook(err):', ' print("EasyBuild FAIL: %s" % err)', @@ -188,7 +190,8 @@ def test_run_hook(self): run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make -j 3"]) run_hook('build', hooks, post_step_hook=True, args=[None]) run_hook('install', hooks, pre_step_hook=True, args=[None]) - run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make install"], kwargs={}) + res = run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make install"], kwargs={}) + self.assertEqual(res, "sudo make install") run_hook('install', hooks, post_step_hook=True, args=[None]) run_hook('extensions', hooks, pre_step_hook=True, args=[None]) for _ in range(3): From 2d6f1a951110243aa94ff25ada5313bbd9160b47 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 11 Aug 2023 20:22:33 +0200 Subject: [PATCH 059/312] pass working directory to pre run_shell_cmd hook + pass output, exit code, and working directory of command to post run_shell_cmd hook --- easybuild/tools/run.py | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 7d6e66ccf5..d8d99f28ce 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -238,10 +238,10 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True if with_hooks: hooks = load_hooks(build_option('hooks')) - hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd]) + hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'work_dir': os.getcwd()}) if isinstance(hook_res, string_type): cmd, old_cmd = hook_res, cmd - self.log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd) + _log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd) _log.info('running cmd: %s ' % cmd) try: @@ -250,9 +250,6 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True except OSError as err: raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err) - if with_hooks: - run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd]) - if inp: proc.stdin.write(inp.encode()) proc.stdin.close() @@ -261,7 +258,7 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True return (proc, cmd, cwd, start_time, cmd_log) else: return complete_cmd(proc, cmd, cwd, start_time, cmd_log, log_ok=log_ok, log_all=log_all, simple=simple, - regexp=regexp, stream_output=stream_output, trace=trace) + regexp=regexp, stream_output=stream_output, trace=trace, with_hook=with_hooks) def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, output_read_size=1024, output=''): @@ -306,7 +303,7 @@ def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, out def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False, simple=False, - regexp=True, stream_output=None, trace=True, output=''): + regexp=True, stream_output=None, trace=True, output='', with_hook=True): """ Complete running of command represented by passed subprocess.Popen instance. @@ -321,6 +318,7 @@ def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) :param stream_output: enable streaming command output to stdout :param trace: print command being executed as part of trace output + :param with_hook: trigger post run_shell_cmd hooks (if defined) """ # use small read size when streaming output, to make it stream more fluently # read size should not be too small though, to avoid too much overhead @@ -356,6 +354,15 @@ def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False sys.stdout.write(output) stdouterr += output + if with_hook: + hooks = load_hooks(build_option('hooks')) + run_hook_kwargs = { + 'exit_code': ec, + 'output': stdouterr, + 'work_dir': os.getcwd(), + } + run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs) + if trace: trace_msg("command completed: exit %s, ran in %s" % (ec, time_str_since(start_time))) @@ -499,10 +506,14 @@ def check_answers_list(answers): # - this needs asynchronous stdout hooks = load_hooks(build_option('hooks')) - hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'interactive': True}) + run_hook_kwargs = { + 'interactive': True, + 'work_dir': os.getcwd(), + } + hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs=run_hook_kwargs) if isinstance(hook_res, string_type): cmd, old_cmd = hook_res, cmd - self.log.info("Interactive command to run was changed by pre-%s hook: '%s' (was: '%s')", + _log.info("Interactive command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd) # # Log command output @@ -619,7 +630,12 @@ def get_proc(): except IOError as err: _log.debug("runqanda cmd %s: remaining data read failed: %s", cmd, err) - run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs={'interactive': True}) + run_hook_kwargs.update({ + 'interactive': True, + 'exit_code': ec, + 'output': stdout_err, + }) + run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs) if trace: trace_msg("interactive command completed: exit %s, ran in %s" % (ec, time_str_since(start_time))) From 1f92ccc6930611d7db8dbb18b77dbc6d1cb2610a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 11 Aug 2023 21:26:55 +0200 Subject: [PATCH 060/312] add test for running shell commands with pre/post run_cmd_shell hooks in place --- test/framework/run.py | 56 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/test/framework/run.py b/test/framework/run.py index f97d3b240f..1a93ff1a75 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -39,6 +39,7 @@ import subprocess import sys import tempfile +import textwrap import time from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -47,6 +48,7 @@ import easybuild.tools.asyncprocess as asyncprocess import easybuild.tools.utilities from easybuild.tools.build_log import EasyBuildError, init_logging, stop_logging +from easybuild.tools.config import update_build_option from easybuild.tools.filetools import adjust_permissions, read_file, write_file from easybuild.tools.run import check_async_cmd, check_log_for_errors, complete_cmd, get_output_from_process from easybuild.tools.run import parse_log_for_error, run_cmd, run_cmd_qa @@ -736,6 +738,60 @@ def test_check_log_for_errors(self): expected_msg = "Found 1 potential error(s) in command output (output: the process crashed with 0)" self.assertIn(expected_msg, read_file(logfile)) + def test_run_cmd_with_hooks(self): + """ + Test running command with run_cmd with pre/post run_shell_cmd hooks in place. + """ + cwd = os.getcwd() + + hooks_file = os.path.join(self.test_prefix, 'my_hooks.py') + hooks_file_txt = textwrap.dedent(""" + def pre_run_shell_cmd_hook(cmd, *args, **kwargs): + work_dir = kwargs['work_dir'] + if kwargs.get('interactive'): + print("pre-run hook interactive '%s' in %s" % (cmd, work_dir)) + else: + print("pre-run hook '%s' in %s" % (cmd, work_dir)) + if not cmd.startswith('echo'): + cmds = cmd.split(';') + return '; '.join(cmds[:-1] + ["echo " + cmds[-1].lstrip()]) + + def post_run_shell_cmd_hook(cmd, *args, **kwargs): + exit_code = kwargs.get('exit_code') + output = kwargs.get('output') + work_dir = kwargs['work_dir'] + if kwargs.get('interactive'): + msg = "post-run hook interactive '%s'" % cmd + else: + msg = "post-run hook '%s'" % cmd + msg += " (exit code: %s, output: '%s')" % (exit_code, output) + print(msg) + """) + write_file(hooks_file, hooks_file_txt) + update_build_option('hooks', hooks_file) + + with self.mocked_stdout_stderr(): + run_cmd("make") + stdout = self.get_stdout() + + expected_stdout = '\n'.join([ + "pre-run hook 'make' in %s" % cwd, + "post-run hook 'echo make' (exit code: 0, output: 'make\n')", + '', + ]) + self.assertEqual(stdout, expected_stdout) + + with self.mocked_stdout_stderr(): + run_cmd_qa("sleep 2; make", qa={}) + stdout = self.get_stdout() + + expected_stdout = '\n'.join([ + "pre-run hook interactive 'sleep 2; make' in %s" % cwd, + "post-run hook interactive 'sleep 2; echo make' (exit code: 0, output: 'make\n')", + '', + ]) + self.assertEqual(stdout, expected_stdout) + def suite(): """ returns all the testcases in this module """ From f0ef6272a37dad11c2fc4d05168e155c5161893b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 11 Aug 2023 21:27:40 +0200 Subject: [PATCH 061/312] extend test_toy_build_hooks with run_shell_cmd hooks --- .../sandbox/easybuild/easyblocks/t/toy.py | 8 ++- test/framework/toy_build.py | 69 ++++++++++++++++++- 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index 88ff864454..c4614e3333 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -34,7 +34,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extensioneasyblock import ExtensionEasyBlock -from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.environment import setvar from easybuild.tools.filetools import mkdir, write_file from easybuild.tools.modules import get_software_root, get_software_version @@ -122,7 +122,11 @@ def build_step(self, name=None, cfg=None): name = self.name cmd = compose_toy_build_cmd(self.cfg, name, cfg['prebuildopts'], cfg['buildopts']) - run_cmd(cmd) + # purposely run build command without checking exit code; + # we rely on this in test_toy_build_hooks + (out, ec) = run_cmd(cmd, log_ok=False, log_all=False) + if ec: + print_warning("Command '%s' failed, but we'll ignore it..." % cmd) def install_step(self, name=None): """Install toy.""" diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 52aa18aae9..934438a549 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -58,6 +58,7 @@ from easybuild.tools.modules import Lmod from easybuild.tools.py2vs3 import reload, string_type from easybuild.tools.run import run_cmd +from easybuild.tools.utilities import nub from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.version import VERSION as EASYBUILD_VERSION @@ -2885,6 +2886,9 @@ def test_toy_build_hooks(self): hooks_file = os.path.join(self.test_prefix, 'my_hooks.py') hooks_file_txt = textwrap.dedent(""" import os + from easybuild.tools.filetools import change_dir, copy_file + + TOY_COMP_CMD = "gcc toy.c -o toy" def start_hook(): print('start hook triggered') @@ -2908,6 +2912,9 @@ def post_install_hook(self): print('in post-install hook for %s v%s' % (self.name, self.version)) print(', '.join(sorted(os.listdir(self.installdir)))) + copy_of_toy = os.path.join(self.start_dir, 'copy_of_toy') + copy_file(copy_of_toy, os.path.join(self.installdir, 'bin')) + def module_write_hook(self, module_path, module_txt): print('in module-write hook hook for %s' % os.path.basename(module_path)) return module_txt.replace('Toy C program, 100% toy.', 'Not a toy anymore') @@ -2920,6 +2927,24 @@ def pre_sanitycheck_hook(self): def end_hook(): print('end hook triggered, all done!') + + def pre_run_shell_cmd_hook(cmd, *args, **kwargs): + if cmd.strip() == TOY_COMP_CMD: + print("pre_run_shell_cmd_hook triggered for '%s'" % cmd) + # 'copy_toy_file' command doesn't exist, but don't worry, + # this problem will be fixed in post_run_shell_cmd_hook + cmd += " && copy_toy_file toy copy_of_toy" + return cmd + + def post_run_shell_cmd_hook(cmd, *args, **kwargs): + exit_code = kwargs['exit_code'] + output = kwargs['output'] + work_dir = kwargs['work_dir'] + if cmd.strip().startswith(TOY_COMP_CMD) and exit_code: + cwd = change_dir(work_dir) + copy_file('toy', 'copy_of_toy') + change_dir(cwd) + print("'%s' command failed (exit code %s), but I fixed it!" % (cmd, exit_code)) """) write_file(hooks_file, hooks_file_txt) @@ -2936,7 +2961,9 @@ def end_hook(): if get_module_syntax() == 'Lua': toy_mod_file += '.lua' - self.assertEqual(stderr, '') + warnings = nub([x for x in stderr.strip().splitlines() if x]) + self.assertEqual(warnings, ["WARNING: Command ' gcc toy.c -o toy ' failed, but we'll ignore it..."]) + # parse hook is triggered 3 times: once for main install, and then again for each extension; # module write hook is triggered 5 times: # - before installing extensions @@ -2950,10 +2977,22 @@ def end_hook(): toy 0.0 ['%(name)s-%(version)s.tar.gz'] echo toy + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... == Running pre-configure hook... pre-configure: toy.source: True + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... == Running post-configure hook... post-configure: toy.source: False + == Running pre-run_shell_cmd hook... + pre_run_shell_cmd_hook triggered for ' gcc toy.c -o toy ' + == Running post-run_shell_cmd hook... + ' gcc toy.c -o toy && copy_toy_file toy copy_of_toy' command failed (exit code 127), but I fixed it! == Running post-install hook... in post-install hook for toy v0.0 bin, lib @@ -2967,16 +3006,40 @@ def end_hook(): toy 0.0 ['%(name)s-%(version)s.tar.gz'] echo toy + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... == Running post-single_extension hook... installing of extension bar is done! + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... + == Running pre-run_shell_cmd hook... + pre_run_shell_cmd_hook triggered for ' gcc toy.c -o toy ' + == Running post-run_shell_cmd hook... + ' gcc toy.c -o toy && copy_toy_file toy copy_of_toy' command failed (exit code 127), but I fixed it! == Running post-single_extension hook... installing of extension toy is done! + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... == Running pre-sanitycheck hook... pre_sanity_check_hook == Running module_write hook... in module-write hook hook for {mod_name} + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... == Running module_write hook... in module-write hook hook for {mod_name} + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... == Running module_write hook... in module-write hook hook for {mod_name} == Running module_write hook... @@ -2989,6 +3052,10 @@ def end_hook(): toy_mod = read_file(toy_mod_file) self.assertIn('Not a toy anymore', toy_mod) + toy_bin_dir = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin') + toy_bins = sorted(os.listdir(toy_bin_dir)) + self.assertEqual(toy_bins, ['bar', 'copy_of_toy', 'toy']) + def test_toy_multi_deps(self): """Test installation of toy easyconfig that uses multi_deps.""" test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') From 6a1e5b8ef452ca559cd90d30a9bf697430e03273 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 11 Aug 2023 22:04:03 +0200 Subject: [PATCH 062/312] trivial code style fix --- easybuild/tools/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index d8d99f28ce..8916d80795 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -514,7 +514,7 @@ def check_answers_list(answers): if isinstance(hook_res, string_type): cmd, old_cmd = hook_res, cmd _log.info("Interactive command to run was changed by pre-%s hook: '%s' (was: '%s')", - RUN_SHELL_CMD, cmd, old_cmd) + RUN_SHELL_CMD, cmd, old_cmd) # # Log command output if cmd_log: From 29dbb7175bf439bd0115145481f37890ae0f9771 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 12 Aug 2023 09:59:22 +0200 Subject: [PATCH 063/312] omit '== Running ... hook' messages in test_toy_build_hooks by not using --debug --- test/framework/toy_build.py | 55 +++---------------------------------- 1 file changed, 4 insertions(+), 51 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 934438a549..a7e32971a6 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -155,7 +155,7 @@ def check_toy(self, installpath, outtxt, name='toy', version='0.0', versionprefi def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True, fails=False, verbose=True, raise_error=False, test_report=None, name='toy', versionsuffix='', testing=True, - raise_systemexit=False, force=True, test_report_regexs=None): + raise_systemexit=False, force=True, test_report_regexs=None, debug=True): """Perform a toy build.""" if extra_args is None: extra_args = [] @@ -169,10 +169,11 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True '--sourcepath=%s' % self.test_sourcepath, '--buildpath=%s' % self.test_buildpath, '--installpath=%s' % self.test_installpath, - '--debug', '--unittest-file=%s' % self.logfile, '--robot=%s' % os.pathsep.join([self.test_buildpath, os.path.dirname(__file__)]), ] + if debug: + args.append('--debug') if force: args.append('--force') if tmpdir is not None: @@ -2950,7 +2951,7 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): self.mock_stderr(True) self.mock_stdout(True) - self.test_toy_build(ec_file=test_ec, extra_args=['--hooks=%s' % hooks_file], raise_error=True) + self.test_toy_build(ec_file=test_ec, extra_args=['--hooks=%s' % hooks_file], raise_error=True, debug=False) stderr = self.get_stderr() stdout = self.get_stdout() self.mock_stderr(False) @@ -2971,80 +2972,32 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): # - for final module file # - for devel module file expected_output = textwrap.dedent(""" - == Running start hook... start hook triggered - == Running parse hook for test.eb... toy 0.0 ['%(name)s-%(version)s.tar.gz'] echo toy - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running pre-configure hook... pre-configure: toy.source: True - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running post-configure hook... post-configure: toy.source: False - == Running pre-run_shell_cmd hook... pre_run_shell_cmd_hook triggered for ' gcc toy.c -o toy ' - == Running post-run_shell_cmd hook... ' gcc toy.c -o toy && copy_toy_file toy copy_of_toy' command failed (exit code 127), but I fixed it! - == Running post-install hook... in post-install hook for toy v0.0 bin, lib - == Running module_write hook... in module-write hook hook for {mod_name} - == Running parse hook... toy 0.0 ['%(name)s-%(version)s.tar.gz'] echo toy - == Running parse hook... toy 0.0 ['%(name)s-%(version)s.tar.gz'] echo toy - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running post-single_extension hook... installing of extension bar is done! - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running pre-run_shell_cmd hook... pre_run_shell_cmd_hook triggered for ' gcc toy.c -o toy ' - == Running post-run_shell_cmd hook... ' gcc toy.c -o toy && copy_toy_file toy copy_of_toy' command failed (exit code 127), but I fixed it! - == Running post-single_extension hook... installing of extension toy is done! - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running pre-sanitycheck hook... pre_sanity_check_hook - == Running module_write hook... in module-write hook hook for {mod_name} - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running module_write hook... in module-write hook hook for {mod_name} - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running module_write hook... in module-write hook hook for {mod_name} - == Running module_write hook... in module-write hook hook for {mod_name} - == Running end hook... end hook triggered, all done! """).strip().format(mod_name=os.path.basename(toy_mod_file)) self.assertEqual(stdout.strip(), expected_output) From 6c5b8bff57ee7c09544889194c545bb1482e32dd Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 12 Aug 2023 10:32:59 +0100 Subject: [PATCH 064/312] add build_info_msg easyconfig parameter --- easybuild/framework/easyblock.py | 4 ++++ easybuild/framework/easyconfig/default.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index c85104d687..55d73d3ba5 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -4174,6 +4174,10 @@ def build_and_install_one(ecdict, init_env): dry_run_msg('', silent=silent) print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent) + if ecdict['ec']['build_info_msg']: + msg = "This easyconfig provides the following build information:\n\n%s\n" + print_msg(msg % ecdict['ec']['build_info_msg'], log=_log, silent=silent) + if dry_run: # print note on interpreting dry run output (argument is reference to location of dry run messages) print_dry_run_note('below', silent=silent) diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index b37c3660f0..dd91229d1e 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -228,6 +228,8 @@ 'buildstats': [None, "A list of dicts with build statistics", OTHER], 'deprecated': [False, "String specifying reason why this easyconfig file is deprecated " "and will be archived in the next major release of EasyBuild", OTHER], + 'build_info_msg': [None, "String with information to be printed to stdout and logged during the building " + "of the easyconfig", OTHER], } From a3cde1a8b0634c759b77236b926c2bb6e5f25049 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 12 Aug 2023 12:21:51 +0200 Subject: [PATCH 065/312] reset cached hooks in tests to avoid test hooks leaking into other tests --- test/framework/hooks.py | 10 +++++++++- test/framework/toy_build.py | 3 +++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/test/framework/hooks.py b/test/framework/hooks.py index 1be55b8ac7..2d150ae869 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -32,7 +32,7 @@ from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner -import easybuild.tools.hooks +import easybuild.tools.hooks # so we can reset cached hooks from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import remove_file, write_file from easybuild.tools.hooks import find_hook, load_hooks, run_hook, verify_hooks @@ -81,6 +81,14 @@ def setUp(self): ]) write_file(self.test_hooks_pymod, test_hooks_pymod_txt) + def tearDown(self): + """Cleanup.""" + + # reset cached hooks + easybuild.tools.hooks._cached_hooks.clear() + + super(HooksTest, self).tearDown() + def test_load_hooks(self): """Test for load_hooks function.""" diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index a7e32971a6..f61a4be33c 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -101,6 +101,9 @@ def tearDown(self): del sys.modules['easybuild.easyblocks.toytoy'] del sys.modules['easybuild.easyblocks.generic.toy_extension'] + # reset cached hooks + easybuild.tools.hooks._cached_hooks.clear() + super(ToyBuildTest, self).tearDown() # remove logs From 5491a621a9f76b27366240666704b61102efb7f8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 9 Aug 2023 20:29:21 +0200 Subject: [PATCH 066/312] remove non-working run_cmd/run_cmd_qa/parse_log_for_error functions from filetools --- easybuild/tools/filetools.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 80a6ba6560..20a803321b 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2253,21 +2253,6 @@ def decode_class_name(name): return decode_string(name) -def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None): - """NO LONGER SUPPORTED: use run_cmd from easybuild.tools.run instead""" - _log.nosupport("run_cmd was moved from easybuild.tools.filetools to easybuild.tools.run", '2.0') - - -def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None): - """NO LONGER SUPPORTED: use run_cmd_qa from easybuild.tools.run instead""" - _log.nosupport("run_cmd_qa was moved from easybuild.tools.filetools to easybuild.tools.run", '2.0') - - -def parse_log_for_error(txt, regExp=None, stdout=True, msg=None): - """NO LONGER SUPPORTED: use parse_log_for_error from easybuild.tools.run instead""" - _log.nosupport("parse_log_for_error was moved from easybuild.tools.filetools to easybuild.tools.run", '2.0') - - def det_size(path): """ Determine total size of given filepath (in bytes). From aaa61e6621966135d66fd4304cc85784bff8d572 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 9 Aug 2023 21:24:57 +0200 Subject: [PATCH 067/312] implement support in 'run' function for running command in different working directory --- easybuild/tools/run.py | 4 ++-- test/framework/run.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index c972277b00..5129ad1fa3 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -135,7 +135,7 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, """ # temporarily raise a NotImplementedError until all options are implemented - if any((work_dir, stream_output, asynchronous)): + if any((stream_output, asynchronous)): raise NotImplementedError if qa_patterns or qa_wait_patterns: @@ -187,7 +187,7 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, _log.info(f"Running command '{cmd_str}' in {work_dir}") proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=stderr, check=fail_on_error, - env=env, input=stdin, shell=shell, executable=executable) + cwd=work_dir, env=env, input=stdin, shell=shell, executable=executable) # return output as a regular string rather than a byte sequence (and non-UTF-8 characters get stripped out) output = proc.stdout.decode('utf-8', 'ignore') diff --git a/test/framework/run.py b/test/framework/run.py index 4cd6821569..c8195cc3a7 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -342,6 +342,48 @@ def test_run_bis(self): self.assertEqual(len(res.output), len("hello\n" * 300)) self.assertEqual(res.exit_code, 0) + def test_run_cmd_work_dir(self): + """ + Test running command in specific directory with run_cmd function. + """ + orig_wd = os.getcwd() + self.assertFalse(os.path.samefile(orig_wd, self.test_prefix)) + + test_dir = os.path.join(self.test_prefix, 'test') + for fn in ('foo.txt', 'bar.txt'): + write_file(os.path.join(test_dir, fn), 'test') + + with self.mocked_stdout_stderr(): + (out, ec) = run_cmd("ls | sort", path=test_dir) + + self.assertEqual(ec, 0) + self.assertEqual(out, 'bar.txt\nfoo.txt\n') + + self.assertTrue(os.path.samefile(orig_wd, os.getcwd())) + + def test_run_work_dir(self): + """ + Test running command in specific directory with run function. + """ + orig_wd = os.getcwd() + self.assertFalse(os.path.samefile(orig_wd, self.test_prefix)) + + test_dir = os.path.join(self.test_prefix, 'test') + for fn in ('foo.txt', 'bar.txt'): + write_file(os.path.join(test_dir, fn), 'test') + + cmd = "ls | sort" + with self.mocked_stdout_stderr(): + res = run(cmd, work_dir=test_dir) + + self.assertEqual(res.cmd, cmd) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, 'bar.txt\nfoo.txt\n') + self.assertEqual(res.stderr, None) + self.assertEqual(res.work_dir, test_dir) + + self.assertTrue(os.path.samefile(orig_wd, os.getcwd())) + def test_run_cmd_log_output(self): """Test run_cmd with log_output enabled""" with self.mocked_stdout_stderr(): From 5f05c4aa96deae74f9924c408b93377c100580d7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 9 Aug 2023 22:03:06 +0200 Subject: [PATCH 068/312] switch to run function in filetools module --- easybuild/tools/filetools.py | 107 ++++++++++++++++++----------------- test/framework/filetools.py | 4 +- 2 files changed, 58 insertions(+), 53 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 20a803321b..aee769f611 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -62,11 +62,11 @@ import urllib.request as std_urllib from easybuild.base import fancylogger -from easybuild.tools import run # import build_log must stay, to use of EasyBuildLog from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning from easybuild.tools.config import ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN, build_option, install_path from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar +from easybuild.tools.run import run from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars, trace_msg try: @@ -445,7 +445,7 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced """ if not os.path.isfile(fn) and not build_option('extended_dry_run'): - raise EasyBuildError("Can't extract file %s: no such file", fn) + raise EasyBuildError(f"Can't extract file {fn}: no such file") mkdir(dest, parents=True) @@ -453,24 +453,24 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced abs_dest = os.path.abspath(dest) # change working directory - _log.debug("Unpacking %s in directory %s", fn, abs_dest) + _log.debug(f"Unpacking {fn} in directory {abs_dest}") cwd = change_dir(abs_dest) if cmd: # complete command template with filename cmd = cmd % fn - _log.debug("Using specified command to unpack %s: %s", fn, cmd) + _log.debug("Using specified command to unpack {fn}: {cmd}") else: cmd = extract_cmd(fn, overwrite=overwrite) - _log.debug("Using command derived from file extension to unpack %s: %s", fn, cmd) + _log.debug("Using command derived from file extension to unpack {fn}: {cmd}") if not cmd: - raise EasyBuildError("Can't extract file %s with unknown filetype", fn) + raise EasyBuildError("Can't extract file {fn} with unknown filetype") if extra_options: - cmd = "%s %s" % (cmd, extra_options) + cmd = f"{cmd} {extra_options}" - run.run_cmd(cmd, simple=True, force_in_dry_run=forced, trace=trace) + run(cmd, in_dry_run=True, hidden=not trace) # note: find_base_dir also changes into the base dir! base_dir = find_base_dir() @@ -479,7 +479,7 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced # change back to where we came from (unless that was a non-existing directory) if not change_into_dir: if cwd is None: - raise EasyBuildError("Can't change back to non-existing directory after extracting %s in %s", fn, dest) + raise EasyBuildError(f"Can't change back to non-existing directory after extracting {fn} in {dest}") else: change_dir(cwd) @@ -1547,27 +1547,27 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git=False if build_option('extended_dry_run'): # skip checking of files in dry run mode patch_filename = os.path.basename(patch_file) - dry_run_msg("* applying patch file %s" % patch_filename, silent=build_option('silent')) + dry_run_msg(f"* applying patch file {patch_filename}", silent=build_option('silent')) elif not os.path.isfile(patch_file): - raise EasyBuildError("Can't find patch %s: no such file", patch_file) + raise EasyBuildError(f"Can't find patch {patch_file}: no such file") elif fn and not os.path.isfile(fn): - raise EasyBuildError("Can't patch file %s: no such file", fn) + raise EasyBuildError(f"Can't patch file {fn}: no such file") # copy missing files if copy: if build_option('extended_dry_run'): - dry_run_msg(" %s copied to %s" % (patch_file, dest), silent=build_option('silent')) + dry_run_msg(f" {patch_file} copied to {dest}", silent=build_option('silent')) else: copy_file(patch_file, dest) - _log.debug("Copied patch %s to dir %s" % (patch_file, dest)) + _log.debug(f"Copied patch {patch_file} to dir {dest}") # early exit, work is done after copying return True elif not os.path.isdir(dest): - raise EasyBuildError("Can't patch directory %s: no such directory", dest) + raise EasyBuildError(f"Can't patch directory {dest}: no such directory") # use absolute paths abs_patch_file = os.path.abspath(patch_file) @@ -1582,14 +1582,14 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git=False patch_subextension = os.path.splitext(patch_stem)[1] if patch_subextension == ".patch": workdir = tempfile.mkdtemp(prefix='eb-patch-') - _log.debug("Extracting the patch to: %s", workdir) + _log.debug(f"Extracting the patch to: {workdir}") # extracting the patch extracted_dir = extract_file(abs_patch_file, workdir, change_into_dir=False) abs_patch_file = os.path.join(extracted_dir, patch_stem) if use_git: verbose = '--verbose ' if build_option('debug') else '' - patch_cmd = "git apply %s%s" % (verbose, abs_patch_file) + patch_cmd = f"git apply {verbose}{abs_patch_file}" else: if level is None and build_option('extended_dry_run'): level = '' @@ -1602,27 +1602,30 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git=False patched_files = det_patched_files(path=abs_patch_file) if not patched_files: - raise EasyBuildError("Can't guess patchlevel from patch %s: no testfile line found in patch", - abs_patch_file) + msg = f"Can't guess patchlevel from patch {abs_patch_file}: no testfile line found in patch" + raise EasyBuildError(msg) level = guess_patch_level(patched_files, abs_dest) if level is None: # level can also be 0 (zero), so don't use "not level" # no match - raise EasyBuildError("Can't determine patch level for patch %s from directory %s", patch_file, abs_dest) + raise EasyBuildError(f"Can't determine patch level for patch {patch_file} from directory {abs_dest}") else: - _log.debug("Guessed patch level %d for patch %s" % (level, patch_file)) + _log.debug(f"Guessed patch level {level} for patch {patch_file}") else: - _log.debug("Using specified patch level %d for patch %s" % (level, patch_file)) + _log.debug(f"Using specified patch level {level} for patch {patch_file}") backup_option = '-b ' if build_option('backup_patched_files') else '' - patch_cmd = "patch " + backup_option + "-p%s -i %s" % (level, abs_patch_file) + patch_cmd = f"patch {backup_option} -p{level} -i {abs_patch_file}" - out, ec = run.run_cmd(patch_cmd, simple=False, path=abs_dest, log_ok=False, trace=False) + res = run(patch_cmd, fail_on_error=False, hidden=True, work_dir=abs_dest) + + if res.exit_code: + msg = f"Couldn't apply patch file {patch_file}. " + msg += f"Process exited with code {res.exit_code}: {res.output}" + raise EasyBuildError(msg) - if ec: - raise EasyBuildError("Couldn't apply patch file %s. Process exited with code %s: %s", patch_file, ec, out) return True @@ -2148,7 +2151,7 @@ def move_logs(src_logfile, target_logfile): try: # there may be multiple log files, due to log rotation - app_logs = glob.glob('%s*' % src_logfile) + app_logs = glob.glob(f'{src_logfile}*') for app_log in app_logs: # retain possible suffix new_log_path = target_logfile + app_log[src_logfile_len:] @@ -2159,11 +2162,11 @@ def move_logs(src_logfile, target_logfile): # move log to target path move_file(app_log, new_log_path) - _log.info("Moved log file %s to %s" % (src_logfile, new_log_path)) + _log.info(f"Moved log file {src_logfile} to {new_log_path}") if zip_log_cmd: - run.run_cmd("%s %s" % (zip_log_cmd, new_log_path)) - _log.info("Zipped log %s using '%s'", new_log_path, zip_log_cmd) + run(f"{zip_log_cmd} {new_log_path}") + _log.info(f"Zipped log {new_log_path} using '{zip_log_cmd}'") except (IOError, OSError) as err: raise EasyBuildError("Failed to move log file(s) %s* to new log file %s*: %s", @@ -2577,7 +2580,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): """ # sanity check on git_config value being passed if not isinstance(git_config, dict): - raise EasyBuildError("Found unexpected type of value for 'git_config' argument: %s" % type(git_config)) + raise EasyBuildError("Found unexpected type of value for 'git_config' argument: {type(git_config)}") # Making a copy to avoid modifying the object with pops git_config = git_config.copy() @@ -2591,7 +2594,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): # input validation of git_config dict if git_config: - raise EasyBuildError("Found one or more unexpected keys in 'git_config' specification: %s", git_config) + raise EasyBuildError("Found one or more unexpected keys in 'git_config' specification: {git_config}") if not repo_name: raise EasyBuildError("repo_name not specified in git_config parameter") @@ -2628,14 +2631,14 @@ def get_source_tarball_from_git(filename, targetdir, git_config): # checkout is done separately below for specific commits clone_cmd.append('--no-checkout') - clone_cmd.append('%s/%s.git' % (url, repo_name)) + clone_cmd.append(f'{url}/{repo_name}.git') if clone_into: - clone_cmd.append('%s' % clone_into) + clone_cmd.append(clone_into) tmpdir = tempfile.mkdtemp() - cwd = change_dir(tmpdir) - run.run_cmd(' '.join(clone_cmd), log_all=True, simple=True, regexp=False, trace=False) + + run(' '.join(clone_cmd), hidden=True, work_dir=tmpdir) # If the clone is done into a specified name, change repo_name if clone_into: @@ -2647,43 +2650,45 @@ def get_source_tarball_from_git(filename, targetdir, git_config): if recursive: checkout_cmd.extend(['&&', 'git', 'submodule', 'update', '--init', '--recursive']) - run.run_cmd(' '.join(checkout_cmd), log_all=True, simple=True, regexp=False, path=repo_name, trace=False) + work_dir = os.path.join(tmpdir, repo_name) if repo_name else tmpdir + run(' '.join(checkout_cmd), work_dir=work_dir, hidden=True) elif not build_option('extended_dry_run'): # If we wanted to get a tag make sure we actually got a tag and not a branch with the same name # This doesn't make sense in dry-run mode as we don't have anything to check - cmd = 'git describe --exact-match --tags HEAD' - # Note: Disable logging to also disable the error handling in run_cmd - (out, ec) = run.run_cmd(cmd, log_ok=False, log_all=False, regexp=False, path=repo_name, trace=False) - if ec != 0 or tag not in out.splitlines(): - print_warning('Tag %s was not downloaded in the first try due to %s/%s containing a branch' - ' with the same name. You might want to alert the maintainers of %s about that issue.', - tag, url, repo_name, repo_name) + cmd = "git describe --exact-match --tags HEAD" + work_dir = os.path.join(tmpdir, repo_name) if repo_name else tmpdir + res = run(cmd, fail_on_error=False, work_dir=work_dir, hidden=True) + + if res.exit_code != 0 or tag not in res.output.splitlines(): + msg = f"Tag {tag} was not downloaded in the first try due to {url}/{repo_name} containing a branch" + msg += f" with the same name. You might want to alert the maintainers of {repo_name} about that issue." + print_warning(msg) + cmds = [] if not keep_git_dir: # make the repo unshallow first; # this is equivalent with 'git fetch -unshallow' in Git 1.8.3+ # (first fetch seems to do nothing, unclear why) - cmds.append('git fetch --depth=2147483647 && git fetch --depth=2147483647') + cmds.append("git fetch --depth=2147483647 && git fetch --depth=2147483647") - cmds.append('git checkout refs/tags/' + tag) + cmds.append(f"git checkout refs/tags/{tag}") # Clean all untracked files, e.g. from left-over submodules - cmds.append('git clean --force -d -x') + cmds.append("git clean --force -d -x") if recursive: - cmds.append('git submodule update --init --recursive') + cmds.append("git submodule update --init --recursive") for cmd in cmds: - run.run_cmd(cmd, log_all=True, simple=True, regexp=False, path=repo_name, trace=False) + run(cmd, work_dir=work_dir, hidden=True) # create an archive and delete the git repo directory if keep_git_dir: tar_cmd = ['tar', 'cfvz', targetpath, repo_name] else: tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name] - run.run_cmd(' '.join(tar_cmd), log_all=True, simple=True, regexp=False, trace=False) + run(' '.join(tar_cmd), work_dir=tmpdir, hidden=True) # cleanup (repo_name dir does not exist in dry run mode) - change_dir(cwd) remove(tmpdir) return targetpath diff --git a/test/framework/filetools.py b/test/framework/filetools.py index fcaebe16d4..d4eb45b7c9 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2831,7 +2831,7 @@ def run_check(): r' running command "git clone --no-checkout %(git_repo)s"', r" \(in /.*\)", r' running command "git checkout 8456f86 && git submodule update --init --recursive"', - r" \(in testrepository\)", + r" \(in /.*/testrepository\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in /.*\)", ]) % git_repo @@ -2842,7 +2842,7 @@ def run_check(): r' running command "git clone --no-checkout %(git_repo)s"', r" \(in /.*\)", r' running command "git checkout 8456f86"', - r" \(in testrepository\)", + r" \(in /.*/testrepository\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in /.*\)", ]) % git_repo From ed127b68bec2169d8e802b504dd54968844391f9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 11 Aug 2023 18:43:01 +0200 Subject: [PATCH 069/312] also test use of run when using EasyBuild as a library + use 'reconfigure' option of set_up_configuration in EasyBuildLibTest.configure helper method --- test/framework/lib.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/test/framework/lib.py b/test/framework/lib.py index 9eed88632f..578f20f46d 100644 --- a/test/framework/lib.py +++ b/test/framework/lib.py @@ -43,7 +43,7 @@ from easybuild.tools.options import set_up_configuration from easybuild.tools.filetools import mkdir from easybuild.tools.modules import modules_tool -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run, run_cmd class EasyBuildLibTest(TestCase): @@ -67,14 +67,7 @@ def tearDown(self): def configure(self): """Utility function to set up EasyBuild configuration.""" - - # wipe BuildOption singleton instance, so it gets re-created when set_up_configuration is called - if BuildOptions in BuildOptions._instances: - del BuildOptions._instances[BuildOptions] - - self.assertNotIn(BuildOptions, BuildOptions._instances) - set_up_configuration(silent=True) - self.assertIn(BuildOptions, BuildOptions._instances) + set_up_configuration(silent=True, reconfigure=True) def test_run_cmd(self): """Test use of run_cmd function in the context of using EasyBuild framework as a library.""" @@ -92,8 +85,24 @@ def test_run_cmd(self): self.assertEqual(ec, 0) self.assertEqual(out, 'hello\n') + def test_run(self): + """Test use of runfunction in the context of using EasyBuild framework as a library.""" + + error_pattern = r"Undefined build option: .*" + error_pattern += r" Make sure you have set up the EasyBuild configuration using set_up_configuration\(\)" + self.assertErrorRegex(EasyBuildError, error_pattern, run, "echo hello") + + self.configure() + + # runworks fine if set_up_configuration was called first + self.mock_stdout(True) + res = run("echo hello") + self.mock_stdout(False) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, 'hello\n') + def test_mkdir(self): - """Test use of run_cmd function in the context of using EasyBuild framework as a library.""" + """Test use of mkdir function in the context of using EasyBuild framework as a library.""" test_dir = os.path.join(self.tmpdir, 'test123') From ecbaf34414d6cf0b1b8110880a62fe15bc97a159 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 12 Aug 2023 18:54:05 +0200 Subject: [PATCH 070/312] remove crash hook (since it doesn't work) --- easybuild/main.py | 5 +---- easybuild/tools/hooks.py | 2 -- test/framework/options.py | 1 - 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 24f994da14..1a9aa736a7 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -69,7 +69,7 @@ from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, merge_pr, new_branch_github, new_pr from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr -from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, CRASH, START, END, CANCEL, FAIL +from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, START, END, CANCEL, FAIL from easybuild.tools.hooks import load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import opts_dict_to_eb_opts, set_up_configuration, use_color @@ -760,6 +760,3 @@ def prepare_main(args=None, logfile=None, testing=None): except KeyboardInterrupt as err: run_hook(CANCEL, hooks, args=[err]) print_error("Cancelled by user: %s" % err) - except Exception as err: - run_hook(CRASH, hooks, args=[err]) - print_error("Encountered an unrecoverable error: %s" % err) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 7a104d6277..785e255171 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -67,7 +67,6 @@ END = 'end' CANCEL = 'cancel' -CRASH = 'crash' FAIL = 'fail' PRE_PREF = 'pre_' @@ -106,7 +105,6 @@ POST_PREF + BUILD_AND_INSTALL_LOOP, END, CANCEL, - CRASH, FAIL, ] KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES] diff --git a/test/framework/options.py b/test/framework/options.py index 2ef6488d16..545d18cdda 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -722,7 +722,6 @@ def test_avail_hooks(self): " post_build_and_install_loop_hook", " end_hook", " cancel_hook", - " crash_hook", " fail_hook", '', ]) From c48615e5a05662e018216266d91f04c4b0df074c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 12 Aug 2023 19:26:45 +0200 Subject: [PATCH 071/312] add test to check whether build_info_msg works as expected --- test/framework/toy_build.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 52aa18aae9..40e967fcf0 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3900,6 +3900,30 @@ def test_toy_post_install_messages(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def test_toy_build_info_msg(self): + """ + Test use of build info message + """ + test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\nbuild_info_msg = "Are you sure you want to install this toy software?"' + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ec_txt) + + with self.mocked_stdout_stderr(): + self.test_toy_build(ec_file=test_ec, testing=False, verify=False, raise_error=True) + stdout = self.get_stdout() + + pattern = '\n'.join([ + r"== This easyconfig provides the following build information:", + r'', + r"Are you sure you want to install this toy software\?", + ]) + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def suite(): """ return all the tests in this file """ From 5b45c240c00feec269ac4af7e093dee9545a524c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 13 Aug 2023 18:06:16 +0200 Subject: [PATCH 072/312] ignore stdout produced by toy easyblock in test_cleanup_builddir --- test/framework/options.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 78c3a495e6..74877765e5 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2580,7 +2580,8 @@ def test_cleanup_builddir(self): '--force', '--try-amend=prebuildopts=nosuchcommand &&', ] - self.eb_main(args, do_build=True) + with self.mocked_stdout_stderr(): + self.eb_main(args, do_build=True) self.assertExists(toy_buildpath, "Build dir %s is retained after failed build" % toy_buildpath) def test_filter_deps(self): From b45d5e7f94bf0ca19cc829e5e9aa09e3946766a8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 14 Aug 2023 11:13:21 +0200 Subject: [PATCH 073/312] only run command to extract file in dry run mode when forced to do so --- easybuild/tools/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index aee769f611..ee8b32303f 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -470,7 +470,7 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced if extra_options: cmd = f"{cmd} {extra_options}" - run(cmd, in_dry_run=True, hidden=not trace) + run(cmd, in_dry_run=forced, hidden=not trace) # note: find_base_dir also changes into the base dir! base_dir = find_base_dir() From 5505b39260bad91c903612d172b3a597e8a92e25 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 14 Aug 2023 11:22:18 +0200 Subject: [PATCH 074/312] add verbose_dry_run option to enable showing of commands being run without actually running them in dry run mode --- easybuild/tools/filetools.py | 10 +++++----- easybuild/tools/run.py | 7 ++++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index ee8b32303f..fd03cea091 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2638,7 +2638,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): tmpdir = tempfile.mkdtemp() - run(' '.join(clone_cmd), hidden=True, work_dir=tmpdir) + run(' '.join(clone_cmd), hidden=True, verbose_dry_run=True, work_dir=tmpdir) # If the clone is done into a specified name, change repo_name if clone_into: @@ -2651,14 +2651,14 @@ def get_source_tarball_from_git(filename, targetdir, git_config): checkout_cmd.extend(['&&', 'git', 'submodule', 'update', '--init', '--recursive']) work_dir = os.path.join(tmpdir, repo_name) if repo_name else tmpdir - run(' '.join(checkout_cmd), work_dir=work_dir, hidden=True) + run(' '.join(checkout_cmd), work_dir=work_dir, hidden=True, verbose_dry_run=True) elif not build_option('extended_dry_run'): # If we wanted to get a tag make sure we actually got a tag and not a branch with the same name # This doesn't make sense in dry-run mode as we don't have anything to check cmd = "git describe --exact-match --tags HEAD" work_dir = os.path.join(tmpdir, repo_name) if repo_name else tmpdir - res = run(cmd, fail_on_error=False, work_dir=work_dir, hidden=True) + res = run(cmd, fail_on_error=False, work_dir=work_dir, hidden=True, verbose_dry_run=True) if res.exit_code != 0 or tag not in res.output.splitlines(): msg = f"Tag {tag} was not downloaded in the first try due to {url}/{repo_name} containing a branch" @@ -2679,14 +2679,14 @@ def get_source_tarball_from_git(filename, targetdir, git_config): if recursive: cmds.append("git submodule update --init --recursive") for cmd in cmds: - run(cmd, work_dir=work_dir, hidden=True) + run(cmd, work_dir=work_dir, hidden=True, verbose_dry_run=True) # create an archive and delete the git repo directory if keep_git_dir: tar_cmd = ['tar', 'cfvz', targetpath, repo_name] else: tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name] - run(' '.join(tar_cmd), work_dir=tmpdir, hidden=True) + run(' '.join(tar_cmd), work_dir=tmpdir, hidden=True, verbose_dry_run=True) # cleanup (repo_name dir does not exist in dry run mode) remove(tmpdir) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 5129ad1fa3..b5f61e72a9 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -108,7 +108,7 @@ def cache_aware_func(cmd, *args, **kwargs): @run_cache def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, - hidden=False, in_dry_run=False, work_dir=None, shell=True, + hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, shell=True, output_file=False, stream_output=False, asynchronous=False, qa_patterns=None, qa_wait_patterns=None): """ @@ -119,7 +119,8 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, :param stdin: input to be sent to stdin (nothing if set to None) :param env: environment to use to run command (if None, inherit current process environment) :param hidden: do not show command in terminal output (when using --trace, or with --extended-dry-run / -x) - :param in_dry_run: also run command in dry run mode + :param in_dry_run: also run command in dry run mode (overrules 'hidden') + :param verbose_dry_run: show that command is run in dry run mode :param work_dir: working directory to run command in (current working directory if None) :param shell: execute command through bash shell (enabled by default) :param output_file: collect command output in temporary output file @@ -162,7 +163,7 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, # early exit in 'dry run' mode, after printing the command that would be run (unless 'hidden' is enabled) if not in_dry_run and build_option('extended_dry_run'): - if not hidden: + if not hidden or verbose_dry_run: silent = build_option('silent') msg = f" running command \"{cmd_str}\"\n" msg += f" (in {work_dir})" From 4e2d495fea0c953f2e4ae74ebde1d0f574ffa829 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 14 Aug 2023 11:28:20 +0200 Subject: [PATCH 075/312] tweak error pattern in test_toy_extension_extract_cmd --- test/framework/toy_build.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 2192b0589c..eb8a417b24 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1429,10 +1429,12 @@ def test_toy_extension_extract_cmd(self): ]) write_file(test_ec, test_ec_txt) - error_pattern = "unzip .*/bar-0.0.tar.gz.* exited with exit code [1-9]" + error_pattern = "unzip .*/bar-0.0.tar.gz.* returned non-zero exit status" with self.mocked_stdout_stderr(): - self.assertErrorRegex(EasyBuildError, error_pattern, self._test_toy_build, ec_file=test_ec, - raise_error=True, verbose=False) + # for now, we expect subprocess.CalledProcessError, but eventually 'run' function will + # do proper error reporting + self.assertErrorRegex(EasyBuildError, error_pattern, + self._test_toy_build, ec_file=test_ec, raise_error=True, verbose=False) def test_toy_extension_sources_git_config(self): """Test install toy that includes extensions with 'sources' spec including 'git_config'.""" From 886d7bf2bb7bc76902edae69ed67b78af695c01d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 15 Aug 2023 16:26:30 +0200 Subject: [PATCH 076/312] Add `makedirs(..., exits_ok)` for Python2 --- easybuild/tools/py2vs3/py2.py | 10 ++++++++++ easybuild/tools/py2vs3/py3.py | 1 + test/framework/filetools.py | 11 +++++++++++ 3 files changed, 22 insertions(+) diff --git a/easybuild/tools/py2vs3/py2.py b/easybuild/tools/py2vs3/py2.py index 99f8cb0f4a..08a3b52b72 100644 --- a/easybuild/tools/py2vs3/py2.py +++ b/easybuild/tools/py2vs3/py2.py @@ -33,8 +33,10 @@ """ # these are not used here, but imported from here in other places import ConfigParser as configparser # noqa +import errno import imp import json +import os import subprocess import time import urllib2 as std_urllib # noqa @@ -115,3 +117,11 @@ def sort_looseversions(looseversions): # with Python 2, we can safely use 'sorted' on LooseVersion instances # (but we can't in Python 3, see https://bugs.python.org/issue14894) return sorted(looseversions) + + +def makedirs(name, mode=0o777, exist_ok=False): + try: + os.makedirs(name, mode) + except OSError as e: + if not exist_ok or e.errno != errno.EEXIST: + raise diff --git a/easybuild/tools/py2vs3/py3.py b/easybuild/tools/py2vs3/py3.py index 77f786cbec..8d9145179d 100644 --- a/easybuild/tools/py2vs3/py3.py +++ b/easybuild/tools/py2vs3/py3.py @@ -44,6 +44,7 @@ from html.parser import HTMLParser # noqa from itertools import zip_longest from io import StringIO # noqa +from os import makedirs # noqa from string import ascii_letters, ascii_lowercase # noqa from urllib.request import HTTPError, HTTPSHandler, Request, URLError, build_opener, urlopen # noqa from urllib.parse import urlencode # noqa diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 32d72c7b83..14011e2cbc 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -45,6 +45,7 @@ from unittest import TextTestRunner from easybuild.tools import run import easybuild.tools.filetools as ft +import easybuild.tools.py2vs3 as py2vs3 from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import IGNORE, ERROR, build_option, update_build_option from easybuild.tools.multidiff import multidiff @@ -3389,6 +3390,16 @@ def test_set_gid_sticky_bits(self): self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + def test_compat_makedirs(self): + """Test compatibility layer for Python3 os.makedirs""" + name = os.path.join(self.test_prefix, 'folder') + self.assertNotExists(name) + py2vs3.makedirs(name) + self.assertExists(name) + self.assertErrorRegex(Exception, os.path.basename(name), py2vs3.makedirs, name) + py2vs3.makedirs(name, exist_ok=True) # No error + self.assertExists(name) + def test_create_unused_dir(self): """Test create_unused_dir function.""" path = ft.create_unused_dir(self.test_prefix, 'folder') From 22def640ae0bfb96280389e96347a015837aecc3 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 15 Aug 2023 16:32:51 +0200 Subject: [PATCH 077/312] Don't rely on errno Same as the Python3 implementation which has a comment: > Cannot rely on checking for EEXIST, since the operating system could give priority to other errors like EACCES or EROFS --- easybuild/tools/py2vs3/py2.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/py2vs3/py2.py b/easybuild/tools/py2vs3/py2.py index 08a3b52b72..1fec4650b1 100644 --- a/easybuild/tools/py2vs3/py2.py +++ b/easybuild/tools/py2vs3/py2.py @@ -33,7 +33,6 @@ """ # these are not used here, but imported from here in other places import ConfigParser as configparser # noqa -import errno import imp import json import os @@ -122,6 +121,6 @@ def sort_looseversions(looseversions): def makedirs(name, mode=0o777, exist_ok=False): try: os.makedirs(name, mode) - except OSError as e: - if not exist_ok or e.errno != errno.EEXIST: + except OSError: + if not exist_ok or not os.path.isdir(name): raise From 204814347b05ce0b5977783aae59e83ee2e219ef Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 15 Aug 2023 16:36:02 +0200 Subject: [PATCH 078/312] Remove use of Python3 `FileExistsError` Use the new py2vpy3.makedirs compat layer. --- easybuild/tools/filetools.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index d16d4f4778..a5e5f63ec6 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -65,7 +65,7 @@ from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN from easybuild.tools.config import build_option, install_path from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar -from easybuild.tools.py2vs3 import HTMLParser, load_source, std_urllib, string_type +from easybuild.tools.py2vs3 import HTMLParser, load_source, makedirs, std_urllib, string_type from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars, trace_msg try: @@ -1918,15 +1918,9 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): # climb up until we hit an existing path or the empty string (for relative paths) while existing_parent_path and not os.path.exists(existing_parent_path): existing_parent_path = os.path.dirname(existing_parent_path) - os.makedirs(path) + makedirs(path, exist_ok=True) else: os.mkdir(path) - except FileExistsError as err: - if os.path.exists(path): - # This may happen if a parallel build creates the directory after we checked for its existence - _log.debug("Directory creation aborted as it seems it was already created: %s", err) - else: - raise EasyBuildError("Failed to create directory %s: %s", path, err) except OSError as err: raise EasyBuildError("Failed to create directory %s: %s", path, err) From 82902c76289134ec3c32a26c43368c1274fb80ea Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 15 Aug 2023 18:32:13 +0200 Subject: [PATCH 079/312] fix docstring for 'run' function w.r.t. verbose_dry_run option --- easybuild/tools/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index b5f61e72a9..76c6883eb9 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -119,8 +119,8 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, :param stdin: input to be sent to stdin (nothing if set to None) :param env: environment to use to run command (if None, inherit current process environment) :param hidden: do not show command in terminal output (when using --trace, or with --extended-dry-run / -x) - :param in_dry_run: also run command in dry run mode (overrules 'hidden') - :param verbose_dry_run: show that command is run in dry run mode + :param in_dry_run: also run command in dry run mode + :param verbose_dry_run: show that command is run in dry run mode (overrules 'hidden') :param work_dir: working directory to run command in (current working directory if None) :param shell: execute command through bash shell (enabled by default) :param output_file: collect command output in temporary output file From 40f9fe761c1810e45ec407c9343647a3283982e8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 15 Aug 2023 18:37:21 +0200 Subject: [PATCH 080/312] use liberal pattern to check whether makedirs wrapper raises an exception if directory already exists, so the check passes with both Python 2 and 3 --- test/framework/filetools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 14011e2cbc..b87b9359b0 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -3396,7 +3396,8 @@ def test_compat_makedirs(self): self.assertNotExists(name) py2vs3.makedirs(name) self.assertExists(name) - self.assertErrorRegex(Exception, os.path.basename(name), py2vs3.makedirs, name) + # exception is raised because file exists (OSError in Python 2, FileExistsError in Python 3) + self.assertErrorRegex(Exception, '.*', py2vs3.makedirs, name) py2vs3.makedirs(name, exist_ok=True) # No error self.assertExists(name) From f5ec5e57ea1fd64cc4ce7da2f1b6563388c28485 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 07:53:47 +0200 Subject: [PATCH 081/312] use amd64 version of containers to run end2end test --- .github/workflows/end2end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 7594b79e59..63d5013f15 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -15,7 +15,7 @@ jobs: - ubuntu-20.04 fail-fast: false container: - image: ghcr.io/easybuilders/${{ matrix.container }} + image: ghcr.io/easybuilders/${{ matrix.container }}-amd64 steps: - name: Check out the repo uses: actions/checkout@v2 From e46d5415775e816e0895b650bbc38dbfa320e94e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 08:23:15 +0200 Subject: [PATCH 082/312] add --silence-hook-trigger configuration option to supress printing of debug message every time a hook is triggered --- easybuild/tools/config.py | 1 + easybuild/tools/hooks.py | 2 +- easybuild/tools/options.py | 2 ++ test/framework/hooks.py | 72 +++++++++++++++++++++++--------------- 4 files changed, 48 insertions(+), 29 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 3f8acec4f5..6c0a173fe4 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -301,6 +301,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'sequential', 'set_default_module', 'set_gid_bit', + 'silence_hook_trigger', 'skip_extensions', 'skip_test_cases', 'skip_test_step', diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 9c61820090..f454974edb 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -233,7 +233,7 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, if msg is None: msg = "Running %s hook..." % label - if build_option('debug'): + if build_option('debug') and not build_option('silence_hook_trigger'): print_msg(msg) _log.info("Running '%s' hook function (args: %s, keyword args: %s)...", hook.__name__, args, kwargs) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 5acf1780a2..b60e57513d 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -501,6 +501,8 @@ def override_options(self): 'silence-deprecation-warnings': ( "Silence specified deprecation warnings out of (%s)" % ', '.join(all_deprecations), 'strlist', 'extend', []), + 'silence-hook-trigger': ("Supress printing of debug message every time a hook is triggered", + None, 'store_true', False), 'skip-extensions': ("Skip installation of extensions", None, 'store_true', False), 'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'), 'skip-test-step': ("Skip running the test step (e.g. unit tests)", None, 'store_true', False), diff --git a/test/framework/hooks.py b/test/framework/hooks.py index 2d150ae869..152d2352a4 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -34,6 +34,7 @@ import easybuild.tools.hooks # so we can reset cached hooks from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import update_build_option from easybuild.tools.filetools import remove_file, write_file from easybuild.tools.hooks import find_hook, load_hooks, run_hook, verify_hooks @@ -186,33 +187,38 @@ def test_run_hook(self): init_config(build_options={'debug': True}) - self.mock_stdout(True) - self.mock_stderr(True) - run_hook('start', hooks) - run_hook('parse', hooks, args=[''], msg="Running parse hook for example.eb...") - run_hook('build_and_install_loop', hooks, args=[['ec1', 'ec2']], pre_step_hook=True) - run_hook('configure', hooks, pre_step_hook=True, args=[None]) - run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["configure.sh"], kwargs={'interactive': True}) - run_hook('configure', hooks, post_step_hook=True, args=[None]) - run_hook('build', hooks, pre_step_hook=True, args=[None]) - run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make -j 3"]) - run_hook('build', hooks, post_step_hook=True, args=[None]) - run_hook('install', hooks, pre_step_hook=True, args=[None]) - res = run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make install"], kwargs={}) - self.assertEqual(res, "sudo make install") - run_hook('install', hooks, post_step_hook=True, args=[None]) - run_hook('extensions', hooks, pre_step_hook=True, args=[None]) - for _ in range(3): - run_hook('single_extension', hooks, pre_step_hook=True, args=[None]) - run_hook('single_extension', hooks, post_step_hook=True, args=[None]) - run_hook('extensions', hooks, post_step_hook=True, args=[None]) - run_hook('fail', hooks, args=[EasyBuildError('oops')]) - stdout = self.get_stdout() - stderr = self.get_stderr() - self.mock_stdout(False) - self.mock_stderr(False) - - expected_stdout = '\n'.join([ + def run_hooks(): + self.mock_stdout(True) + self.mock_stderr(True) + run_hook('start', hooks) + run_hook('parse', hooks, args=[''], msg="Running parse hook for example.eb...") + run_hook('build_and_install_loop', hooks, args=[['ec1', 'ec2']], pre_step_hook=True) + run_hook('configure', hooks, pre_step_hook=True, args=[None]) + run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["configure.sh"], kwargs={'interactive': True}) + run_hook('configure', hooks, post_step_hook=True, args=[None]) + run_hook('build', hooks, pre_step_hook=True, args=[None]) + run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make -j 3"]) + run_hook('build', hooks, post_step_hook=True, args=[None]) + run_hook('install', hooks, pre_step_hook=True, args=[None]) + res = run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make install"], kwargs={}) + self.assertEqual(res, "sudo make install") + run_hook('install', hooks, post_step_hook=True, args=[None]) + run_hook('extensions', hooks, pre_step_hook=True, args=[None]) + for _ in range(3): + run_hook('single_extension', hooks, pre_step_hook=True, args=[None]) + run_hook('single_extension', hooks, post_step_hook=True, args=[None]) + run_hook('extensions', hooks, post_step_hook=True, args=[None]) + run_hook('fail', hooks, args=[EasyBuildError('oops')]) + stdout = self.get_stdout() + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + + return stdout, stderr + + stdout, stderr = run_hooks() + + expected_stdout_lines = [ "== Running start hook...", "this is triggered at the very beginning", "== Running parse hook for example.eb...", @@ -238,7 +244,17 @@ def test_run_hook(self): "this is run before installing an extension", "== Running fail hook...", "EasyBuild FAIL: 'oops'", - ]) + ] + expected_stdout = '\n'.join(expected_stdout_lines) + + self.assertEqual(stdout.strip(), expected_stdout) + self.assertEqual(stderr, '') + + # test silencing of hook trigger + update_build_option('silence_hook_trigger', True) + stdout, stderr = run_hooks() + + expected_stdout = '\n'.join(x for x in expected_stdout_lines if not x.startswith('== Running')) self.assertEqual(stdout.strip(), expected_stdout) self.assertEqual(stderr, '') From c9d45e4dbad23bfeaf898b1ebd344e254a95c2eb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 08:28:45 +0200 Subject: [PATCH 083/312] update list of containers to use in end2end tests --- .github/workflows/end2end.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 63d5013f15..06a192661f 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -8,11 +8,12 @@ jobs: matrix: container: - centos-7.9 - - centos-8.4 - - fedora-35 - - opensuse-15.3 - - rockylinux-8.5 + - centos-8.5 + - fedora-36 + - opensuse-15.4 + - rockylinux-8.7 - ubuntu-20.04 + - ubuntu-22.04 fail-fast: false container: image: ghcr.io/easybuilders/${{ matrix.container }}-amd64 From 8eeae322b3779497f933abb03bf2ba7b6dc4cd2b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 09:36:06 +0200 Subject: [PATCH 084/312] run unit tests with python2 via CentOS 7.9 container --- .github/workflows/unit_tests_python2.yml | 86 ++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 .github/workflows/unit_tests_python2.yml diff --git a/.github/workflows/unit_tests_python2.yml b/.github/workflows/unit_tests_python2.yml new file mode 100644 index 0000000000..d15b8c240c --- /dev/null +++ b/.github/workflows/unit_tests_python2.yml @@ -0,0 +1,86 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: EasyBuild framework unit tests (python2) +on: [push, pull_request] + +permissions: + contents: read # to fetch code (actions/checkout) + +concurrency: + group: ${{format('{0}:{1}:{2}', github.repository, github.ref, github.workflow)}} + cancel-in-progress: true + +jobs: + test_python2: + runs-on: ubuntu-20.04 + container: + # CentOS 7.9 container that already includes Lmod & co, + # see https://github.com/easybuilders/easybuild-containers + image: ghcr.io/easybuilders/centos-7.9-amd64 + steps: + - uses: actions/checkout@v3 + + - name: install Python packages + run: | + # Python packages + python2 -V + python2 -m pip --version + python2 -m pip install --upgrade pip + python2 -m pip --version + python2 -m pip install -r requirements.txt + # git config is required to make actual git commits (cfr. tests for GitRepository) + git config --global user.name "Travis CI" + git config --global user.email "travis@travis-ci.org" + git config --get-regexp 'user.*' + + - name: install GitHub token (if available) + env: + # token (owned by @boegelbot) with gist permissions (required for some of the tests for GitHub integration); + # this token is not available in pull requests, so tests that require it are skipped in PRs, + # and are only run after the PR gets merged + GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}} + run: | + # tests that require a GitHub token are skipped automatically when no GitHub token is available + if [ ! -z $GITHUB_TOKEN ]; then + python2 -c "import keyring; import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring()); keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; + echo "GitHub token installed!" + else + echo "Installation of GitHub token skipped!" + fi + + - name: install sources + run: | + # install from source distribution tarball, to test release as published on PyPI + python2 setup.py sdist + ls dist + export PREFIX=/tmp/$USER/$GITHUB_SHA + python2 -m pip install --prefix $PREFIX dist/easybuild-framework*tar.gz + + - name: run test suite + env: + EB_VERBOSE: 1 + EB_PYTHON: python2 + run: | + # run tests *outside* of checked out easybuild-framework directory, + # to ensure we're testing installed version (see previous step) + cd $HOME + # initialize environment for modules tool + source /etc/profile.d/z00_lmod.sh + type module + module --version + # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that); + # also pick up changes to $PATH set by sourcing $MOD_INIT + export PREFIX=/tmp/$USER/$GITHUB_SHA + export PATH=$PREFIX/bin:$PATH + export PYTHONPATH=$PREFIX/lib/python2.7/site-packages:$PYTHONPATH + eb --version + # show active EasyBuild configuration + eb --show-config + # gather some useful info on test system + eb --show-system-info + # check GitHub configuration + eb --check-github --github-user=easybuild_test + # create file owned by root but writable by anyone (used by test_copy_file) + sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt + sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt + # run test suite + python2 -O -m test.framework.suite From cf81cec2adc6c17e395da6ebe25d428cf8b651d7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 09:40:08 +0200 Subject: [PATCH 085/312] strip out GC3Pie from requirements.txt before installing Python packages when testing with Python 2 --- .github/workflows/unit_tests_python2.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/unit_tests_python2.yml b/.github/workflows/unit_tests_python2.yml index d15b8c240c..92870724a7 100644 --- a/.github/workflows/unit_tests_python2.yml +++ b/.github/workflows/unit_tests_python2.yml @@ -26,6 +26,8 @@ jobs: python2 -m pip --version python2 -m pip install --upgrade pip python2 -m pip --version + # strip out GC3Pie since installation with ancient setuptools (0.9.8) fails + sed -i '/GC3Pie/d' requirements.txt python2 -m pip install -r requirements.txt # git config is required to make actual git commits (cfr. tests for GitRepository) git config --global user.name "Travis CI" From c335cc9e967d8d2e514791897273d6674a57fafb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 09:48:04 +0200 Subject: [PATCH 086/312] run command via easybuild user and login shell when running test suite with python2 --- .github/workflows/unit_tests_python2.yml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/unit_tests_python2.yml b/.github/workflows/unit_tests_python2.yml index 92870724a7..e508dfc5a6 100644 --- a/.github/workflows/unit_tests_python2.yml +++ b/.github/workflows/unit_tests_python2.yml @@ -43,7 +43,7 @@ jobs: run: | # tests that require a GitHub token are skipped automatically when no GitHub token is available if [ ! -z $GITHUB_TOKEN ]; then - python2 -c "import keyring; import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring()); keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; + sudo -u easybuild python2 -c "import keyring; import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring()); keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; echo "GitHub token installed!" else echo "Installation of GitHub token skipped!" @@ -65,24 +65,21 @@ jobs: # run tests *outside* of checked out easybuild-framework directory, # to ensure we're testing installed version (see previous step) cd $HOME - # initialize environment for modules tool - source /etc/profile.d/z00_lmod.sh - type module - module --version # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that); # also pick up changes to $PATH set by sourcing $MOD_INIT export PREFIX=/tmp/$USER/$GITHUB_SHA export PATH=$PREFIX/bin:$PATH export PYTHONPATH=$PREFIX/lib/python2.7/site-packages:$PYTHONPATH - eb --version + # run EasyBuild command via (non-root) easybuild user + login shell + sudo -u easybuild bash -l -c 'module --version; eb --version' # show active EasyBuild configuration - eb --show-config + sudo -u easybuild bash -l -c 'eb --show-config' # gather some useful info on test system - eb --show-system-info + sudo -u easybuild bash -l -c 'eb --show-system-info' # check GitHub configuration - eb --check-github --github-user=easybuild_test + sudo -u easybuild bash -l -c 'eb --check-github --github-user=easybuild_test' # create file owned by root but writable by anyone (used by test_copy_file) sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt - # run test suite - python2 -O -m test.framework.suite + # run test suite (via easybuild user + login shell) + sudo -u easybuild bash -l -c "python2 -O -m test.framework.suite" From 5deae99370a27ade745df72e08f7e7ab71cb5b51 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 09:18:01 +0200 Subject: [PATCH 087/312] split end2end test in separate steps --- .github/workflows/end2end.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 06a192661f..9ab4605bd0 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -30,7 +30,7 @@ jobs: rm -f develop.tar.gz done - - name: End-to-end test of EasyBuild + - name: Set up environment shell: bash run: | export PATH=$PWD:$PATH @@ -49,15 +49,28 @@ jobs: # tests are run with root privileges, so we need to tell EasyBuild that's OK... export EASYBUILD_ALLOW_USE_AS_ROOT_AND_ACCEPT_CONSEQUENCES=1 + # save full environment to use in subsequent steps, + # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable + while read line; do echo "$line" >> $GITHUB_ENV ; done < <(env) + + - name: Run commands to check test environment + shell: bash + run: | cmds=( + "whoami" + "pwd" "env | sort" "eb --version" "eb --show-system-info" "eb --check-eb-deps" "eb --show-config" - "eb bzip2-1.0.8.eb --trace --robot" ) for cmd in "${cmds[@]}"; do echo ">>> $cmd" eval "$cmd" done + + - name: End-to-end test of installing bzip2 with EasyBuild + shell: bash + run: | + eb bzip2-1.0.8.eb --trace --robot From e21dfc04225914a9836941a890747cfb26329a76 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 11:29:21 +0200 Subject: [PATCH 088/312] run commands in end2end test workflow via login shell by using 'bash -l -c' --- .github/workflows/end2end.yml | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 9ab4605bd0..1c70d20e8e 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -33,25 +33,13 @@ jobs: - name: Set up environment shell: bash run: | - export PATH=$PWD:$PATH - export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop - - # initialize environment (for Lmod) - if [ -f /etc/profile.d/modules.sh ]; then - # for Rocky, CentOS 8, Fedora - echo ">>> sourcing /etc/profile.d/modules.sh" - . /etc/profile.d/modules.sh - else - echo ">>> sourcing /etc/profile.d/*lmod*.sh" - . /etc/profile.d/*lmod*.sh - fi + # set environment variables to be used in subsequent steps, + # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable + echo "PATH=$PWD:$PATH" >> $GITHUB_ENV + echo "PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" >> $GITHUB_ENV # tests are run with root privileges, so we need to tell EasyBuild that's OK... - export EASYBUILD_ALLOW_USE_AS_ROOT_AND_ACCEPT_CONSEQUENCES=1 - - # save full environment to use in subsequent steps, - # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable - while read line; do echo "$line" >> $GITHUB_ENV ; done < <(env) + echo "EASYBUILD_ALLOW_USE_AS_ROOT_AND_ACCEPT_CONSEQUENCES=1" >> $GITHUB_ENV - name: Run commands to check test environment shell: bash @@ -67,10 +55,10 @@ jobs: ) for cmd in "${cmds[@]}"; do echo ">>> $cmd" - eval "$cmd" + bash -l -c "$cmd" done - name: End-to-end test of installing bzip2 with EasyBuild shell: bash run: | - eb bzip2-1.0.8.eb --trace --robot + bash -l -c "eb bzip2-1.0.8.eb --trace --robot" From 34846525b186856891a49b77446fe26f1b286e3f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 11:46:45 +0200 Subject: [PATCH 089/312] update to end2end test workflow to actions/checkout@v3 --- .github/workflows/end2end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 1c70d20e8e..5a0f2d768d 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -19,7 +19,7 @@ jobs: image: ghcr.io/easybuilders/${{ matrix.container }}-amd64 steps: - name: Check out the repo - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: download and unpack easyblocks and easyconfigs repositories run: | From a47d43b1e939de49e45f83af73b137b9f3943cec Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 11:49:37 +0200 Subject: [PATCH 090/312] set $PATH directory in command being run in login shell --- .github/workflows/end2end.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 5a0f2d768d..7b38978796 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -34,8 +34,8 @@ jobs: shell: bash run: | # set environment variables to be used in subsequent steps, - # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable - echo "PATH=$PWD:$PATH" >> $GITHUB_ENV + # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable; + # not: we can't extend $PATH this way, because it will not always be picked up in login shell used in later steps echo "PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" >> $GITHUB_ENV # tests are run with root privileges, so we need to tell EasyBuild that's OK... @@ -55,10 +55,10 @@ jobs: ) for cmd in "${cmds[@]}"; do echo ">>> $cmd" - bash -l -c "$cmd" + bash -l -c "export PATH=$PWD:$PATH; $cmd" done - name: End-to-end test of installing bzip2 with EasyBuild shell: bash run: | - bash -l -c "eb bzip2-1.0.8.eb --trace --robot" + bash -l -c "export PATH=$PWD:$PATH; eb bzip2-1.0.8.eb --trace --robot" From c941493fc0cc4679ee170ebf35fbd959d3f97ba0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 13:44:02 +0200 Subject: [PATCH 091/312] prepend updates to $PATH and $PYTHONPATH in commands being run via `sudo -u easybuild bash -l -c ...` --- .github/workflows/unit_tests_python2.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/unit_tests_python2.yml b/.github/workflows/unit_tests_python2.yml index e508dfc5a6..971ac4ca36 100644 --- a/.github/workflows/unit_tests_python2.yml +++ b/.github/workflows/unit_tests_python2.yml @@ -65,21 +65,19 @@ jobs: # run tests *outside* of checked out easybuild-framework directory, # to ensure we're testing installed version (see previous step) cd $HOME - # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that); - # also pick up changes to $PATH set by sourcing $MOD_INIT + # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that) export PREFIX=/tmp/$USER/$GITHUB_SHA - export PATH=$PREFIX/bin:$PATH - export PYTHONPATH=$PREFIX/lib/python2.7/site-packages:$PYTHONPATH + ENV_CMDS="export PATH=$PREFIX/bin:$PATH; export PYTHONPATH=$PREFIX/lib/python2.7/site-packages:$PYTHONPATH" # run EasyBuild command via (non-root) easybuild user + login shell - sudo -u easybuild bash -l -c 'module --version; eb --version' + sudo -u easybuild bash -l -c "${ENV_CMDS}; module --version; eb --version" # show active EasyBuild configuration - sudo -u easybuild bash -l -c 'eb --show-config' + sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --show-config" # gather some useful info on test system - sudo -u easybuild bash -l -c 'eb --show-system-info' + sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --show-system-info" # check GitHub configuration - sudo -u easybuild bash -l -c 'eb --check-github --github-user=easybuild_test' + sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --check-github --github-user=easybuild_test" # create file owned by root but writable by anyone (used by test_copy_file) sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt # run test suite (via easybuild user + login shell) - sudo -u easybuild bash -l -c "python2 -O -m test.framework.suite" + sudo -u easybuild bash -l -c "${ENV_CMDS}; python2 -O -m test.framework.suite" From 7ea111ec18f78783c72cca3d88f2d54e6b782e25 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 13:47:28 +0200 Subject: [PATCH 092/312] run commands in end2end test worfklow via 'sudo -u easybuild' --- .github/workflows/end2end.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 7b38978796..1207c3888d 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -35,11 +35,8 @@ jobs: run: | # set environment variables to be used in subsequent steps, # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable; - # not: we can't extend $PATH this way, because it will not always be picked up in login shell used in later steps - echo "PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" >> $GITHUB_ENV - - # tests are run with root privileges, so we need to tell EasyBuild that's OK... - echo "EASYBUILD_ALLOW_USE_AS_ROOT_AND_ACCEPT_CONSEQUENCES=1" >> $GITHUB_ENV + ENV_CMDS="export PATH=$PWD:$PATH; export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" + echo "ENV_CMDS='${ENV_CMDS}'" >> $GITHUB_ENV - name: Run commands to check test environment shell: bash @@ -55,10 +52,10 @@ jobs: ) for cmd in "${cmds[@]}"; do echo ">>> $cmd" - bash -l -c "export PATH=$PWD:$PATH; $cmd" + sudo -u easybuild bash -l -c "${ENV_CMDS}; $cmd" done - name: End-to-end test of installing bzip2 with EasyBuild shell: bash run: | - bash -l -c "export PATH=$PWD:$PATH; eb bzip2-1.0.8.eb --trace --robot" + sudo -u easybuild bash -l -c "${ENV_CMDS}; eb bzip2-1.0.8.eb --trace --robot" From fbbc048c965a02dcd64bb711960c3de3e1570509 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 14:25:58 +0200 Subject: [PATCH 093/312] fix updating $PATH and $PYTHONPATH before running command in end2end test workflow --- .github/workflows/end2end.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 1207c3888d..14aa31b9a1 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -35,8 +35,8 @@ jobs: run: | # set environment variables to be used in subsequent steps, # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable; - ENV_CMDS="export PATH=$PWD:$PATH; export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" - echo "ENV_CMDS='${ENV_CMDS}'" >> $GITHUB_ENV + EB_ENV="PATH=$PWD:$PATH PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" + echo "EB_ENV='${EB_ENV}'" >> $GITHUB_ENV - name: Run commands to check test environment shell: bash @@ -52,10 +52,10 @@ jobs: ) for cmd in "${cmds[@]}"; do echo ">>> $cmd" - sudo -u easybuild bash -l -c "${ENV_CMDS}; $cmd" + sudo -u easybuild bash -l -c "${EB_ENV} $cmd" done - name: End-to-end test of installing bzip2 with EasyBuild shell: bash run: | - sudo -u easybuild bash -l -c "${ENV_CMDS}; eb bzip2-1.0.8.eb --trace --robot" + sudo -u easybuild bash -l -c "${EB_ENV} eb bzip2-1.0.8.eb --trace --robot" From a042318bb5de8cc2dfd93294dc69db1b11aeb050 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 14:39:14 +0200 Subject: [PATCH 094/312] tweak eval_quoted_string helper function in test_quote_py_str to be compatible with both (old versions of) Python 2.7 & 3.x --- test/framework/easyconfig.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index e17f1c3ca7..6ce2d09650 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -2147,10 +2147,13 @@ def eval_quoted_string(quoted_val, val): """ globals = dict() try: - exec('res = %s' % quoted_val, globals) + # this is needlessly complicated because we can't use 'exec' here without potentially running + # into a SyntaxError bug in old Python 2.7 versions (for example when running the tests in CentOS 7.9) + # cfr. https://stackoverflow.com/questions/4484872/why-doesnt-exec-work-in-a-function-with-a-subfunction + edict = {}; eval(compile('res = %s' % quoted_val, '', 'exec'), globals, edict) except Exception as e: # pylint: disable=broad-except self.fail('Failed to evaluate %s (from %s): %s' % (quoted_val, val, e)) - return globals['res'] + return edict['res'] def assertEqual_unquoted(quoted_val, val): """Assert that evaluating the quoted_val yields the val""" From 0d49930a9ab187a1fcdf8674a0357523ee8615fd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 14:54:46 +0200 Subject: [PATCH 095/312] use source script rather than modifying $GITHUB_ENV --- .github/workflows/end2end.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 14aa31b9a1..ba6b02ea8e 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -33,10 +33,9 @@ jobs: - name: Set up environment shell: bash run: | - # set environment variables to be used in subsequent steps, - # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable; - EB_ENV="PATH=$PWD:$PATH PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" - echo "EB_ENV='${EB_ENV}'" >> $GITHUB_ENV + # collect environment variables to be set in subsequent steps in script that can be sourced + echo "export PATH=$PWD:$PATH" > /tmp/eb_env + echo "export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" >> /tmp/eb_env - name: Run commands to check test environment shell: bash @@ -52,10 +51,10 @@ jobs: ) for cmd in "${cmds[@]}"; do echo ">>> $cmd" - sudo -u easybuild bash -l -c "${EB_ENV} $cmd" + sudo -u easybuild bash -l -c "source /tmp/eb_env; $cmd" done - name: End-to-end test of installing bzip2 with EasyBuild shell: bash run: | - sudo -u easybuild bash -l -c "${EB_ENV} eb bzip2-1.0.8.eb --trace --robot" + sudo -u easybuild bash -l -c "source /tmp/eb_env; eb bzip2-1.0.8.eb --trace --robot" From b93eb7ddf44eab7816d999ae7c73a2c70fc43ce1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 18:21:21 +0200 Subject: [PATCH 096/312] silence deprecation warning for using Python 2 in workflow to run tests with Python 2.7 in CentOS 7.9 container --- .github/workflows/unit_tests_python2.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/unit_tests_python2.yml b/.github/workflows/unit_tests_python2.yml index 971ac4ca36..06f3097436 100644 --- a/.github/workflows/unit_tests_python2.yml +++ b/.github/workflows/unit_tests_python2.yml @@ -58,9 +58,6 @@ jobs: python2 -m pip install --prefix $PREFIX dist/easybuild-framework*tar.gz - name: run test suite - env: - EB_VERBOSE: 1 - EB_PYTHON: python2 run: | # run tests *outside* of checked out easybuild-framework directory, # to ensure we're testing installed version (see previous step) @@ -68,6 +65,7 @@ jobs: # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that) export PREFIX=/tmp/$USER/$GITHUB_SHA ENV_CMDS="export PATH=$PREFIX/bin:$PATH; export PYTHONPATH=$PREFIX/lib/python2.7/site-packages:$PYTHONPATH" + ENV_CMDS="${ENV_CMDS}; export EB_VERBOSE=1; export EB_PYTHON=python2; export TEST_EASYBUILD_SILENCE_DEPRECATION_WARNINGS=python2" # run EasyBuild command via (non-root) easybuild user + login shell sudo -u easybuild bash -l -c "${ENV_CMDS}; module --version; eb --version" # show active EasyBuild configuration From 773f9b97bea7615e29d9350e83aab9c1d1a61c6b Mon Sep 17 00:00:00 2001 From: Rovanion Luckey Date: Wed, 16 Aug 2023 18:54:50 +0200 Subject: [PATCH 097/312] Add support for fine grained Github tokens --- easybuild/tools/github.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index bc3a6b3e27..3bb21a6b95 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -2243,15 +2243,16 @@ def install_github_token(github_user, silent=False): def validate_github_token(token, github_user): """ Check GitHub token: - * see if it conforms expectations (only [a-f]+[0-9] characters, length of 40) - * see if it can be used for authenticated access + * see if it conforms expectations (character classes depending on type, length of 40-93), + * see if it can be used for authenticated access. """ # cfr. https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/ token_regex = re.compile('^ghp_[a-zA-Z0-9]{36}$') token_regex_old_format = re.compile('^[0-9a-f]{40}$') + # https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/about-authentication-to-github#githubs-token-formats + token_regex_fine_grained = re.compile('github_pat_[a-zA-Z0-9_]{82}') - # token should be 40 characters long, and only contain characters in [0-9a-f] - sanity_check = bool(token_regex.match(token)) + sanity_check = bool(token_regex.match(token)) or bool(token_regex_fine_grained.match(token)) if sanity_check: _log.info("Sanity check on token passed") else: From 1e8dee8e8c0ee064f8f244c99c06a54669bce91e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 19:20:44 +0200 Subject: [PATCH 098/312] fix code style issues in helper function in test_quote_py_str --- test/framework/easyconfig.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 6ce2d09650..ab1924088d 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -2145,15 +2145,15 @@ def eval_quoted_string(quoted_val, val): Helper function to sanity check we can use the quoted string in Python contexts. Returns the evaluated (i.e. unquoted) string """ - globals = dict() + scope = dict() try: # this is needlessly complicated because we can't use 'exec' here without potentially running # into a SyntaxError bug in old Python 2.7 versions (for example when running the tests in CentOS 7.9) # cfr. https://stackoverflow.com/questions/4484872/why-doesnt-exec-work-in-a-function-with-a-subfunction - edict = {}; eval(compile('res = %s' % quoted_val, '', 'exec'), globals, edict) - except Exception as e: # pylint: disable=broad-except - self.fail('Failed to evaluate %s (from %s): %s' % (quoted_val, val, e)) - return edict['res'] + eval(compile('res = %s' % quoted_val, '', 'exec'), dict(), scope) + except Exception as err: # pylint: disable=broad-except + self.fail('Failed to evaluate %s (from %s): %s' % (quoted_val, val, err)) + return scope['res'] def assertEqual_unquoted(quoted_val, val): """Assert that evaluating the quoted_val yields the val""" From 905d919f8c420528b57a63711d4b482bc149fdaf Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 20:49:10 +0200 Subject: [PATCH 099/312] configure Git for easybuild user in workflow for running test suite with Python 2 --- .github/workflows/unit_tests_python2.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unit_tests_python2.yml b/.github/workflows/unit_tests_python2.yml index 06f3097436..1b921ee83c 100644 --- a/.github/workflows/unit_tests_python2.yml +++ b/.github/workflows/unit_tests_python2.yml @@ -30,9 +30,9 @@ jobs: sed -i '/GC3Pie/d' requirements.txt python2 -m pip install -r requirements.txt # git config is required to make actual git commits (cfr. tests for GitRepository) - git config --global user.name "Travis CI" - git config --global user.email "travis@travis-ci.org" - git config --get-regexp 'user.*' + sudo -u easybuild git config --global user.name "GitHub Actions" + sudo -u easybuild git config --global user.email "actions@github.com" + sudo -u easybuild git config --get-regexp 'user.*' - name: install GitHub token (if available) env: From 6c026b3ca7eff2b94e0e767ec46c6c48a2890d95 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 20:49:27 +0200 Subject: [PATCH 100/312] fix :avail pattern in test_debug_lmod --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 219aa4a39a..f4d331519a 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5232,7 +5232,7 @@ def test_debug_lmod(self): init_config(build_options={'debug_lmod': True}) out = self.modtool.run_module('avail', return_output=True) - for pattern in [r"^Lmod version", r"^lmod\(--terse -D avail\)\{", "Master:avail"]: + for pattern in [r"^Lmod version", r"^lmod\(--terse -D avail\)\{", ":avail"]: regex = re.compile(pattern, re.M) self.assertTrue(regex.search(out), "Pattern '%s' found in: %s" % (regex.pattern, out)) else: From c46cc6c96489ab635f452bc5d11c205b308e4869 Mon Sep 17 00:00:00 2001 From: Jasper <65227842+jfgrimm@users.noreply.github.com> Date: Mon, 21 Aug 2023 11:43:15 +0100 Subject: [PATCH 101/312] Update test/framework/lib.py --- test/framework/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/lib.py b/test/framework/lib.py index 578f20f46d..dd1d27c41b 100644 --- a/test/framework/lib.py +++ b/test/framework/lib.py @@ -86,7 +86,7 @@ def test_run_cmd(self): self.assertEqual(out, 'hello\n') def test_run(self): - """Test use of runfunction in the context of using EasyBuild framework as a library.""" + """Test use of run function in the context of using EasyBuild framework as a library.""" error_pattern = r"Undefined build option: .*" error_pattern += r" Make sure you have set up the EasyBuild configuration using set_up_configuration\(\)" From 23e22cd5cdb4ae0b08127dba71465854513d9e15 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 15 Aug 2023 16:39:32 +0200 Subject: [PATCH 102/312] break cyclic import by moving load_source from tools.filetools to tools.hooks --- easybuild/tools/filetools.py | 10 +--------- easybuild/tools/hooks.py | 10 +++++++++- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 8a4892bb7e..20a2c9c620 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -58,7 +58,6 @@ import zlib from functools import partial from html.parser import HTMLParser -from importlib.util import spec_from_file_location, module_from_spec import urllib.request as std_urllib from easybuild.base import fancylogger @@ -66,6 +65,7 @@ from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning from easybuild.tools.config import ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN, build_option, install_path from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar +from easybuild.tools.hooks import load_source from easybuild.tools.run import run from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars, trace_msg @@ -2771,14 +2771,6 @@ def install_fake_vsc(): return fake_vsc_path -def load_source(filename, path): - """Load file as Python module""" - spec = spec_from_file_location(filename, path) - module = module_from_spec(spec) - spec.loader.exec_module(module) - return module - - def get_easyblock_class_name(path): """Make sure file is an easyblock and get easyblock class name""" fn = os.path.basename(path).split('.')[0] diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index b165b3ed71..4534a94656 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -35,7 +35,7 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option -from easybuild.tools.filetools import load_source +from importlib.util import spec_from_file_location, module_from_spec _log = fancylogger.getLogger('hooks', fname=False) @@ -118,6 +118,14 @@ _cached_hooks = {} +def load_source(filename, path): + """Load file as Python module""" + spec = spec_from_file_location(filename, path) + module = module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def load_hooks(hooks_path): """Load defined hooks (if any).""" From 0f6db8125d14e3266bd3b46e70e5e164d06bd0bd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 15 Aug 2023 16:39:44 +0200 Subject: [PATCH 103/312] also trigger pre/post run_shell_cmd hooks in run function --- easybuild/tools/run.py | 19 ++++++++++++++++++- easybuild/tools/systemtools.py | 22 +++++++++++----------- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 7ae81829b8..e670c2f9a8 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -110,7 +110,7 @@ def cache_aware_func(cmd, *args, **kwargs): @run_cache def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, shell=True, - output_file=False, stream_output=False, asynchronous=False, + output_file=False, stream_output=False, asynchronous=False, with_hooks=True, qa_patterns=None, qa_wait_patterns=None): """ Run specified (interactive) shell command, and capture output + exit code. @@ -127,6 +127,7 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, :param output_file: collect command output in temporary output file :param stream_output: stream command output to stdout :param asynchronous: run command asynchronously + :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined) :param qa_patterns: list of 2-tuples with patterns for questions + corresponding answers :param qa_wait_patterns: list of 2-tuples with patterns for non-questions and number of iterations to allow these patterns to match with end out command output @@ -187,6 +188,13 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, stderr = subprocess.PIPE if split_stderr else subprocess.STDOUT + if with_hooks: + hooks = load_hooks(build_option('hooks')) + hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd_str], kwargs={'work_dir': work_dir}) + if isinstance(hook_res, str): + cmd, old_cmd = hook_res, cmd + _log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd) + _log.info(f"Running command '{cmd_str}' in {work_dir}") proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=stderr, check=fail_on_error, cwd=work_dir, env=env, input=stdin, shell=shell, executable=executable) @@ -197,6 +205,15 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, res = RunResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr_output, work_dir=work_dir) + if with_hooks: + run_hook_kwargs = { + 'exit_code': res.exit_code, + 'output': res.output, + 'stderr': res.stderr, + 'work_dir': res.work_dir, + } + run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd_str], kwargs=run_hook_kwargs) + if split_stderr: log_msg = f"Command '{cmd_str}' exited with exit code {res.exit_code}, " log_msg += f"with stdout:\n{res.output}\nstderr:\n{res.stderr}" diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index e6e3de7067..a7f738d5ee 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -274,7 +274,7 @@ def get_avail_core_count(): core_cnt = int(sum(sched_getaffinity())) else: # BSD-type systems - res = run('sysctl -n hw.ncpu', in_dry_run=True, hidden=True) + res = run('sysctl -n hw.ncpu', in_dry_run=True, hidden=True, with_hooks=False) try: if int(res.output) > 0: core_cnt = int(res.output) @@ -311,7 +311,7 @@ def get_total_memory(): elif os_type == DARWIN: cmd = "sysctl -n hw.memsize" _log.debug("Trying to determine total memory size on Darwin via cmd '%s'", cmd) - res = run(cmd, in_dry_run=True, hidden=True) + res = run(cmd, in_dry_run=True, hidden=True, with_hooks=False) if res.exit_code == 0: memtotal = int(res.output.strip()) // (1024**2) @@ -393,14 +393,14 @@ def get_cpu_vendor(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.vendor" - res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True) + res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False) out = res.output.strip() if res.exit_code == 0 and out in VENDOR_IDS: vendor = VENDOR_IDS[out] _log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd)) else: cmd = "sysctl -n machdep.cpu.brand_string" - res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True) + res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False) out = res.output.strip().split(' ')[0] if res.exit_code == 0 and out in CPU_VENDORS: vendor = out @@ -503,7 +503,7 @@ def get_cpu_model(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.brand_string" - res = run(cmd, in_dry_run=True, hidden=True) + res = run(cmd, in_dry_run=True, hidden=True, with_hooks=False) if res.exit_code == 0: model = res.output.strip() _log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model)) @@ -548,7 +548,7 @@ def get_cpu_speed(): elif os_type == DARWIN: cmd = "sysctl -n hw.cpufrequency_max" _log.debug("Trying to determine CPU frequency on Darwin via cmd '%s'" % cmd) - res = run(cmd, in_dry_run=True, hidden=True) + res = run(cmd, in_dry_run=True, hidden=True, with_hooks=False) out = res.output.strip() cpu_freq = None if res.exit_code == 0 and out: @@ -596,7 +596,7 @@ def get_cpu_features(): for feature_set in ['extfeatures', 'features', 'leaf7_features']: cmd = "sysctl -n machdep.cpu.%s" % feature_set _log.debug("Trying to determine CPU features on Darwin via cmd '%s'", cmd) - res = run(cmd, in_dry_run=True, hidden=True, fail_on_error=False) + res = run(cmd, in_dry_run=True, hidden=True, fail_on_error=False, with_hooks=False) if res.exit_code == 0: cpu_feat.extend(res.output.strip().lower().split()) @@ -623,7 +623,7 @@ def get_gpu_info(): try: cmd = "nvidia-smi --query-gpu=gpu_name,driver_version --format=csv,noheader" _log.debug("Trying to determine NVIDIA GPU info on Linux via cmd '%s'", cmd) - res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True) + res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False) if res.exit_code == 0: for line in res.output.strip().split('\n'): nvidia_gpu_info = gpu_info.setdefault('NVIDIA', {}) @@ -641,13 +641,13 @@ def get_gpu_info(): try: cmd = "rocm-smi --showdriverversion --csv" _log.debug("Trying to determine AMD GPU driver on Linux via cmd '%s'", cmd) - res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True) + res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False) if res.exit_code == 0: amd_driver = res.output.strip().split('\n')[1].split(',')[1] cmd = "rocm-smi --showproductname --csv" _log.debug("Trying to determine AMD GPU info on Linux via cmd '%s'", cmd) - res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True) + res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False) if res.exit_code == 0: for line in res.output.strip().split('\n')[1:]: amd_card_series = line.split(',')[1] @@ -893,7 +893,7 @@ def get_tool_version(tool, version_option='--version', ignore_ec=False): Get output of running version option for specific command line tool. Output is returned as a single-line string (newlines are replaced by '; '). """ - res = run(' '.join([tool, version_option]), fail_on_error=False, in_dry_run=True, hidden=True) + res = run(' '.join([tool, version_option]), fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False) if not ignore_ec and res.exit_code: _log.warning("Failed to determine version of %s using '%s %s': %s" % (tool, tool, version_option, res.output)) return UNKNOWN From 4adb564f1ce59ba7124ee4644c1aea3242fc8185 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 15 Aug 2023 16:46:20 +0200 Subject: [PATCH 104/312] fix broken tests --- test/framework/run.py | 3 +++ test/framework/toy_build.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/test/framework/run.py b/test/framework/run.py index 5203d1915e..c12247e471 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1203,6 +1203,9 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): write_file(hooks_file, hooks_file_txt) update_build_option('hooks', hooks_file) + # disable trace output to make checking of generated output produced by hooks easier + update_build_option('trace', False) + with self.mocked_stdout_stderr(): run_cmd("make") stdout = self.get_stdout() diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 948ba6398a..b5e2b88354 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3037,7 +3037,7 @@ def pre_run_shell_cmd_hook(cmd, *args, **kwargs): # 'copy_toy_file' command doesn't exist, but don't worry, # this problem will be fixed in post_run_shell_cmd_hook cmd += " && copy_toy_file toy copy_of_toy" - return cmd + return cmd def post_run_shell_cmd_hook(cmd, *args, **kwargs): exit_code = kwargs['exit_code'] @@ -4051,7 +4051,7 @@ def test_toy_build_info_msg(self): write_file(test_ec, test_ec_txt) with self.mocked_stdout_stderr(): - self.test_toy_build(ec_file=test_ec, testing=False, verify=False, raise_error=True) + self._test_toy_build(ec_file=test_ec, testing=False, verify=False, raise_error=True) stdout = self.get_stdout() pattern = '\n'.join([ From 862962dbbba3f760aca1d16283ab9cf9999209f3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 23 Aug 2023 09:57:38 +0200 Subject: [PATCH 105/312] add + fix test for triggering run_cmd_shell hook in run function --- easybuild/tools/run.py | 28 +++++++++++++++--------- test/framework/run.py | 48 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index e670c2f9a8..cde1a77f61 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -136,6 +136,18 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, - exit_code: exit code of command (integer) - stderr: stderr output if split_stderr is enabled, None otherwise """ + def to_cmd_str(cmd): + """ + Helper function to create string representation of specified command. + """ + if isinstance(cmd, str): + cmd_str = cmd.strip() + elif isinstance(cmd, list): + cmd_str = ' '.join(cmd) + else: + raise EasyBuildError(f"Unknown command type ('{type(cmd)}'): {cmd}") + + return cmd_str # temporarily raise a NotImplementedError until all options are implemented if any((stream_output, asynchronous)): @@ -144,13 +156,6 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, if qa_patterns or qa_wait_patterns: raise NotImplementedError - if isinstance(cmd, str): - cmd_str = cmd.strip() - elif isinstance(cmd, list): - cmd_str = ' '.join(cmd) - else: - raise EasyBuildError(f"Unknown command type ('{type(cmd)}'): {cmd}") - if work_dir is None: work_dir = os.getcwd() @@ -163,6 +168,8 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, else: cmd_out_fp = None + cmd_str = to_cmd_str(cmd) + # early exit in 'dry run' mode, after printing the command that would be run (unless 'hidden' is enabled) if not in_dry_run and build_option('extended_dry_run'): if not hidden or verbose_dry_run: @@ -190,9 +197,10 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, if with_hooks: hooks = load_hooks(build_option('hooks')) - hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd_str], kwargs={'work_dir': work_dir}) - if isinstance(hook_res, str): + hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'work_dir': work_dir}) + if hook_res: cmd, old_cmd = hook_res, cmd + cmd_str = to_cmd_str(cmd) _log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd) _log.info(f"Running command '{cmd_str}' in {work_dir}") @@ -212,7 +220,7 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, 'stderr': res.stderr, 'work_dir': res.work_dir, } - run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd_str], kwargs=run_hook_kwargs) + run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs) if split_stderr: log_msg = f"Command '{cmd_str}' exited with exit code {res.exit_code}, " diff --git a/test/framework/run.py b/test/framework/run.py index c12247e471..b5faf637f4 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1228,6 +1228,54 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): ]) self.assertEqual(stdout, expected_stdout) + def test_run_with_hooks(self): + """ + Test running command with run with pre/post run_shell_cmd hooks in place. + """ + cwd = os.getcwd() + + hooks_file = os.path.join(self.test_prefix, 'my_hooks.py') + hooks_file_txt = textwrap.dedent(""" + def pre_run_shell_cmd_hook(cmd, *args, **kwargs): + work_dir = kwargs['work_dir'] + if kwargs.get('interactive'): + print("pre-run hook interactive '||%s||' in %s" % (cmd, work_dir)) + else: + print("pre-run hook '%s' in %s" % (cmd, work_dir)) + import sys + sys.stderr.write('pre-run hook done\\n') + if not cmd.startswith('echo'): + cmds = cmd.split(';') + return '; '.join(cmds[:-1] + ["echo " + cmds[-1].lstrip()]) + + def post_run_shell_cmd_hook(cmd, *args, **kwargs): + exit_code = kwargs.get('exit_code') + output = kwargs.get('output') + work_dir = kwargs['work_dir'] + if kwargs.get('interactive'): + msg = "post-run hook interactive '%s'" % cmd + else: + msg = "post-run hook '%s'" % cmd + msg += " (exit code: %s, output: '%s')" % (exit_code, output) + print(msg) + """) + write_file(hooks_file, hooks_file_txt) + update_build_option('hooks', hooks_file) + + # disable trace output to make checking of generated output produced by hooks easier + update_build_option('trace', False) + + with self.mocked_stdout_stderr(): + run("make") + stdout = self.get_stdout() + + expected_stdout = '\n'.join([ + "pre-run hook 'make' in %s" % cwd, + "post-run hook 'echo make' (exit code: 0, output: 'make\n')", + '', + ]) + self.assertEqual(stdout, expected_stdout) + def suite(): """ returns all the testcases in this module """ From 3c209b883132b4231af8cfee7ff9dd114a4c2dd0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 24 Aug 2023 10:06:53 +0200 Subject: [PATCH 106/312] fix broken test_toy_build_hooks --- test/framework/toy_build.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index b5e2b88354..2172c6b277 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3032,7 +3032,7 @@ def end_hook(): print('end hook triggered, all done!') def pre_run_shell_cmd_hook(cmd, *args, **kwargs): - if cmd.strip() == TOY_COMP_CMD: + if isinstance(cmd, str) and cmd.strip() == TOY_COMP_CMD: print("pre_run_shell_cmd_hook triggered for '%s'" % cmd) # 'copy_toy_file' command doesn't exist, but don't worry, # this problem will be fixed in post_run_shell_cmd_hook @@ -3043,7 +3043,7 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): exit_code = kwargs['exit_code'] output = kwargs['output'] work_dir = kwargs['work_dir'] - if cmd.strip().startswith(TOY_COMP_CMD) and exit_code: + if isinstance(cmd, str) and cmd.strip().startswith(TOY_COMP_CMD) and exit_code: cwd = change_dir(work_dir) copy_file('toy', 'copy_of_toy') change_dir(cwd) @@ -3051,9 +3051,15 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): """) write_file(hooks_file, hooks_file_txt) + extra_args = [ + '--hooks=%s' % hooks_file, + # disable trace output to make checking of generated output produced by hooks easier + '--disable-trace', + ] + self.mock_stderr(True) self.mock_stdout(True) - self._test_toy_build(ec_file=test_ec, extra_args=['--hooks=%s' % hooks_file], raise_error=True, debug=False) + self._test_toy_build(ec_file=test_ec, extra_args=extra_args, raise_error=True, debug=False) stderr = self.get_stderr() stdout = self.get_stdout() self.mock_stderr(False) @@ -3074,7 +3080,7 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): # - for fake module file being created during sanity check (triggered twice, for main + toy install) # - for final module file # - for devel module file - expected_output = textwrap.dedent(""" + expected_output = textwrap.dedent(f""" start hook triggered toy 0.0 ['%(name)s-%(version)s.tar.gz'] From 36f68afc2b4415391d2e5f7dd1815199d8aa139d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 24 Aug 2023 11:00:37 +0200 Subject: [PATCH 107/312] fix indent for 'return cmd' in pre_run_shell_cmd_hook in test_toy_build_hooks --- test/framework/toy_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 2172c6b277..f8280ba733 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3037,7 +3037,7 @@ def pre_run_shell_cmd_hook(cmd, *args, **kwargs): # 'copy_toy_file' command doesn't exist, but don't worry, # this problem will be fixed in post_run_shell_cmd_hook cmd += " && copy_toy_file toy copy_of_toy" - return cmd + return cmd def post_run_shell_cmd_hook(cmd, *args, **kwargs): exit_code = kwargs['exit_code'] From 6aa3875b9a77b281c068bd1e5526df32cba01571 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 24 Aug 2023 11:02:41 +0200 Subject: [PATCH 108/312] remove test_compat_makedirs since use of py2vs3 is deprecated (and makedirs in py2vs3/py3.py is just os.makedirs, not a wrapper function like it was in py2vs3/py2.py --- test/framework/filetools.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index a93ed7afb6..d4eb45b7c9 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -49,7 +49,6 @@ from urllib import request from easybuild.tools import run import easybuild.tools.filetools as ft -import easybuild.tools.py2vs3 as py2vs3 from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import IGNORE, ERROR, build_option, update_build_option from easybuild.tools.multidiff import multidiff @@ -3411,17 +3410,6 @@ def test_set_gid_sticky_bits(self): self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) - def test_compat_makedirs(self): - """Test compatibility layer for Python3 os.makedirs""" - name = os.path.join(self.test_prefix, 'folder') - self.assertNotExists(name) - py2vs3.makedirs(name) - self.assertExists(name) - # exception is raised because file exists (OSError in Python 2, FileExistsError in Python 3) - self.assertErrorRegex(Exception, '.*', py2vs3.makedirs, name) - py2vs3.makedirs(name, exist_ok=True) # No error - self.assertExists(name) - def test_create_unused_dir(self): """Test create_unused_dir function.""" path = ft.create_unused_dir(self.test_prefix, 'folder') From b849ef832ea45575cc02d082c6d62c517a6600af Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 24 Aug 2023 21:10:58 +0200 Subject: [PATCH 109/312] rename 'run' function to 'run_shell_cmd' --- easybuild/tools/filetools.py | 18 +++--- easybuild/tools/modules.py | 11 ++-- easybuild/tools/run.py | 8 +-- easybuild/tools/systemtools.py | 38 ++++++------- test/framework/lib.py | 10 ++-- test/framework/modules.py | 4 +- test/framework/run.py | 101 +++++++++++++++++---------------- test/framework/systemtools.py | 42 +++++++------- 8 files changed, 117 insertions(+), 115 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 20a2c9c620..f8c733dd8c 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -66,7 +66,7 @@ from easybuild.tools.config import ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN, build_option, install_path from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.hooks import load_source -from easybuild.tools.run import run +from easybuild.tools.run import run_shell_cmd from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars, trace_msg try: @@ -470,7 +470,7 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced if extra_options: cmd = f"{cmd} {extra_options}" - run(cmd, in_dry_run=forced, hidden=not trace) + run_shell_cmd(cmd, in_dry_run=forced, hidden=not trace) # note: find_base_dir also changes into the base dir! base_dir = find_base_dir() @@ -1619,7 +1619,7 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git=False backup_option = '-b ' if build_option('backup_patched_files') else '' patch_cmd = f"patch {backup_option} -p{level} -i {abs_patch_file}" - res = run(patch_cmd, fail_on_error=False, hidden=True, work_dir=abs_dest) + res = run_shell_cmd(patch_cmd, fail_on_error=False, hidden=True, work_dir=abs_dest) if res.exit_code: msg = f"Couldn't apply patch file {patch_file}. " @@ -2165,7 +2165,7 @@ def move_logs(src_logfile, target_logfile): _log.info(f"Moved log file {src_logfile} to {new_log_path}") if zip_log_cmd: - run(f"{zip_log_cmd} {new_log_path}") + run_shell_cmd(f"{zip_log_cmd} {new_log_path}") _log.info(f"Zipped log {new_log_path} using '{zip_log_cmd}'") except (IOError, OSError) as err: @@ -2638,7 +2638,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): tmpdir = tempfile.mkdtemp() - run(' '.join(clone_cmd), hidden=True, verbose_dry_run=True, work_dir=tmpdir) + run_shell_cmd(' '.join(clone_cmd), hidden=True, verbose_dry_run=True, work_dir=tmpdir) # If the clone is done into a specified name, change repo_name if clone_into: @@ -2651,14 +2651,14 @@ def get_source_tarball_from_git(filename, targetdir, git_config): checkout_cmd.extend(['&&', 'git', 'submodule', 'update', '--init', '--recursive']) work_dir = os.path.join(tmpdir, repo_name) if repo_name else tmpdir - run(' '.join(checkout_cmd), work_dir=work_dir, hidden=True, verbose_dry_run=True) + run_shell_cmd(' '.join(checkout_cmd), work_dir=work_dir, hidden=True, verbose_dry_run=True) elif not build_option('extended_dry_run'): # If we wanted to get a tag make sure we actually got a tag and not a branch with the same name # This doesn't make sense in dry-run mode as we don't have anything to check cmd = "git describe --exact-match --tags HEAD" work_dir = os.path.join(tmpdir, repo_name) if repo_name else tmpdir - res = run(cmd, fail_on_error=False, work_dir=work_dir, hidden=True, verbose_dry_run=True) + res = run_shell_cmd(cmd, fail_on_error=False, work_dir=work_dir, hidden=True, verbose_dry_run=True) if res.exit_code != 0 or tag not in res.output.splitlines(): msg = f"Tag {tag} was not downloaded in the first try due to {url}/{repo_name} containing a branch" @@ -2679,14 +2679,14 @@ def get_source_tarball_from_git(filename, targetdir, git_config): if recursive: cmds.append("git submodule update --init --recursive") for cmd in cmds: - run(cmd, work_dir=work_dir, hidden=True, verbose_dry_run=True) + run_shell_cmd(cmd, work_dir=work_dir, hidden=True, verbose_dry_run=True) # create an archive and delete the git repo directory if keep_git_dir: tar_cmd = ['tar', 'cfvz', targetpath, repo_name] else: tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name] - run(' '.join(tar_cmd), work_dir=tmpdir, hidden=True, verbose_dry_run=True) + run_shell_cmd(' '.join(tar_cmd), work_dir=tmpdir, hidden=True, verbose_dry_run=True) # cleanup (repo_name dir does not exist in dry run mode) remove(tmpdir) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 2a32e7fdd4..8e43eff50a 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -50,7 +50,7 @@ from easybuild.tools.environment import ORIG_OS_ENVIRON, restore_env, setvar, unset_env_vars from easybuild.tools.filetools import convert_name, mkdir, normalize_path, path_matches, read_file, which, write_file from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX -from easybuild.tools.run import run +from easybuild.tools.run import run_shell_cmd from easybuild.tools.utilities import get_subclasses, nub # software root/version environment variable name prefixes @@ -312,7 +312,7 @@ def check_module_function(self, allow_mismatch=False, regex=None): output, exit_code = None, 1 else: cmd = "type module" - res = run(cmd, fail_on_error=False, in_dry_run=False, hidden=True) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=False, hidden=True) output, exit_code = res.output, res.exit_code if regex is None: @@ -824,8 +824,8 @@ def run_module(self, *args, **kwargs): cmd_list = self.compose_cmd_list(args) cmd = ' '.join(cmd_list) # note: module commands are always run in dry mode, and are kept hidden in trace and dry run output - res = run(cmd_list, env=environ, fail_on_error=False, shell=False, split_stderr=True, - hidden=True, in_dry_run=True) + res = run_shell_cmd(cmd_list, env=environ, fail_on_error=False, shell=False, split_stderr=True, + hidden=True, in_dry_run=True) # stdout will contain python code (to change environment etc) # stderr will contain text (just like the normal module command) @@ -1426,7 +1426,8 @@ def update(self): cmd = ' '.join(cmd_list) self.log.debug("Running command '%s'...", cmd) - res = run(cmd_list, env=os.environ, fail_on_error=False, shell=False, split_stderr=True, hidden=True) + res = run_shell_cmd(cmd_list, env=os.environ, fail_on_error=False, shell=False, split_stderr=True, + hidden=True) stdout, stderr = res.output, res.stderr if stderr: diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index cde1a77f61..400146aa22 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -108,10 +108,10 @@ def cache_aware_func(cmd, *args, **kwargs): @run_cache -def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, - hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, shell=True, - output_file=False, stream_output=False, asynchronous=False, with_hooks=True, - qa_patterns=None, qa_wait_patterns=None): +def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, + hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, shell=True, + output_file=False, stream_output=False, asynchronous=False, with_hooks=True, + qa_patterns=None, qa_wait_patterns=None): """ Run specified (interactive) shell command, and capture output + exit code. diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index a7f738d5ee..9468e8db13 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -45,7 +45,6 @@ from collections import OrderedDict from ctypes.util import find_library from socket import gethostname -from easybuild.tools.run import subprocess_popen_text # pkg_resources is provided by the setuptools Python package, # which we really want to keep as an *optional* dependency @@ -65,7 +64,7 @@ from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import IGNORE from easybuild.tools.filetools import is_readable, read_file, which -from easybuild.tools.run import run +from easybuild.tools.run import run_shell_cmd, subprocess_popen_text _log = fancylogger.getLogger('systemtools', fname=False) @@ -274,7 +273,7 @@ def get_avail_core_count(): core_cnt = int(sum(sched_getaffinity())) else: # BSD-type systems - res = run('sysctl -n hw.ncpu', in_dry_run=True, hidden=True, with_hooks=False) + res = run_shell_cmd('sysctl -n hw.ncpu', in_dry_run=True, hidden=True, with_hooks=False) try: if int(res.output) > 0: core_cnt = int(res.output) @@ -311,7 +310,7 @@ def get_total_memory(): elif os_type == DARWIN: cmd = "sysctl -n hw.memsize" _log.debug("Trying to determine total memory size on Darwin via cmd '%s'", cmd) - res = run(cmd, in_dry_run=True, hidden=True, with_hooks=False) + res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False) if res.exit_code == 0: memtotal = int(res.output.strip()) // (1024**2) @@ -393,14 +392,14 @@ def get_cpu_vendor(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.vendor" - res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False) out = res.output.strip() if res.exit_code == 0 and out in VENDOR_IDS: vendor = VENDOR_IDS[out] _log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd)) else: cmd = "sysctl -n machdep.cpu.brand_string" - res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False) out = res.output.strip().split(' ')[0] if res.exit_code == 0 and out in CPU_VENDORS: vendor = out @@ -503,7 +502,7 @@ def get_cpu_model(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.brand_string" - res = run(cmd, in_dry_run=True, hidden=True, with_hooks=False) + res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False) if res.exit_code == 0: model = res.output.strip() _log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model)) @@ -548,7 +547,7 @@ def get_cpu_speed(): elif os_type == DARWIN: cmd = "sysctl -n hw.cpufrequency_max" _log.debug("Trying to determine CPU frequency on Darwin via cmd '%s'" % cmd) - res = run(cmd, in_dry_run=True, hidden=True, with_hooks=False) + res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False) out = res.output.strip() cpu_freq = None if res.exit_code == 0 and out: @@ -596,7 +595,7 @@ def get_cpu_features(): for feature_set in ['extfeatures', 'features', 'leaf7_features']: cmd = "sysctl -n machdep.cpu.%s" % feature_set _log.debug("Trying to determine CPU features on Darwin via cmd '%s'", cmd) - res = run(cmd, in_dry_run=True, hidden=True, fail_on_error=False, with_hooks=False) + res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, fail_on_error=False, with_hooks=False) if res.exit_code == 0: cpu_feat.extend(res.output.strip().lower().split()) @@ -623,7 +622,7 @@ def get_gpu_info(): try: cmd = "nvidia-smi --query-gpu=gpu_name,driver_version --format=csv,noheader" _log.debug("Trying to determine NVIDIA GPU info on Linux via cmd '%s'", cmd) - res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False) if res.exit_code == 0: for line in res.output.strip().split('\n'): nvidia_gpu_info = gpu_info.setdefault('NVIDIA', {}) @@ -641,13 +640,13 @@ def get_gpu_info(): try: cmd = "rocm-smi --showdriverversion --csv" _log.debug("Trying to determine AMD GPU driver on Linux via cmd '%s'", cmd) - res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False) if res.exit_code == 0: amd_driver = res.output.strip().split('\n')[1].split(',')[1] cmd = "rocm-smi --showproductname --csv" _log.debug("Trying to determine AMD GPU info on Linux via cmd '%s'", cmd) - res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False) if res.exit_code == 0: for line in res.output.strip().split('\n')[1:]: amd_card_series = line.split(',')[1] @@ -866,7 +865,7 @@ def check_os_dependency(dep): pkg_cmd_flag.get(pkg_cmd), dep, ]) - res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True) found = res.exit_code == 0 if found: break @@ -878,7 +877,7 @@ def check_os_dependency(dep): # try locate if it's available if not found and which('locate'): cmd = 'locate -c --regexp "/%s$"' % dep - res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True) try: found = (res.exit_code == 0 and int(res.output.strip()) > 0) except ValueError: @@ -893,7 +892,8 @@ def get_tool_version(tool, version_option='--version', ignore_ec=False): Get output of running version option for specific command line tool. Output is returned as a single-line string (newlines are replaced by '; '). """ - res = run(' '.join([tool, version_option]), fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False) + res = run_shell_cmd(' '.join([tool, version_option]), fail_on_error=False, in_dry_run=True, + hidden=True, with_hooks=False) if not ignore_ec and res.exit_code: _log.warning("Failed to determine version of %s using '%s %s': %s" % (tool, tool, version_option, res.output)) return UNKNOWN @@ -905,7 +905,7 @@ def get_gcc_version(): """ Process `gcc --version` and return the GCC version. """ - res = run('gcc --version', fail_on_error=False, in_dry_run=True, hidden=True) + res = run_shell_cmd('gcc --version', fail_on_error=False, in_dry_run=True, hidden=True) gcc_ver = None if res.exit_code: _log.warning("Failed to determine the version of GCC: %s", res.output) @@ -961,7 +961,7 @@ def get_linked_libs_raw(path): or None for other types of files. """ - res = run("file %s" % path, fail_on_error=False, hidden=True) + res = run_shell_cmd("file %s" % path, fail_on_error=False, hidden=True) if res.exit_code: fail_msg = "Failed to run 'file %s': %s" % (path, res.output) _log.warning(fail_msg) @@ -996,7 +996,7 @@ def get_linked_libs_raw(path): # take into account that 'ldd' may fail for strange reasons, # like printing 'not a dynamic executable' when not enough memory is available # (see also https://bugzilla.redhat.com/show_bug.cgi?id=1817111) - res = run(linked_libs_cmd, fail_on_error=False, hidden=True) + res = run_shell_cmd(linked_libs_cmd, fail_on_error=False, hidden=True) if res.exit_code == 0: linked_libs_out = res.output else: @@ -1178,7 +1178,7 @@ def get_default_parallelism(): # No cache -> Calculate value from current system values par = get_avail_core_count() # determine max user processes via ulimit -u - res = run("ulimit -u", in_dry_run=True, hidden=True) + res = run_shell_cmd("ulimit -u", in_dry_run=True, hidden=True) try: if res.output.startswith("unlimited"): maxuserproc = 2 ** 32 - 1 diff --git a/test/framework/lib.py b/test/framework/lib.py index dd1d27c41b..a6459b28c3 100644 --- a/test/framework/lib.py +++ b/test/framework/lib.py @@ -43,7 +43,7 @@ from easybuild.tools.options import set_up_configuration from easybuild.tools.filetools import mkdir from easybuild.tools.modules import modules_tool -from easybuild.tools.run import run, run_cmd +from easybuild.tools.run import run_shell_cmd, run_cmd class EasyBuildLibTest(TestCase): @@ -85,18 +85,18 @@ def test_run_cmd(self): self.assertEqual(ec, 0) self.assertEqual(out, 'hello\n') - def test_run(self): - """Test use of run function in the context of using EasyBuild framework as a library.""" + def test_run_shell_cmd(self): + """Test use of run_shell_cmd function in the context of using EasyBuild framework as a library.""" error_pattern = r"Undefined build option: .*" error_pattern += r" Make sure you have set up the EasyBuild configuration using set_up_configuration\(\)" - self.assertErrorRegex(EasyBuildError, error_pattern, run, "echo hello") + self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, "echo hello") self.configure() # runworks fine if set_up_configuration was called first self.mock_stdout(True) - res = run("echo hello") + res = run_shell_cmd("echo hello") self.mock_stdout(False) self.assertEqual(res.exit_code, 0) self.assertEqual(res.output, 'hello\n') diff --git a/test/framework/modules.py b/test/framework/modules.py index d0c330e622..7458f00c41 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -50,7 +50,7 @@ from easybuild.tools.modules import EnvironmentModules, EnvironmentModulesC, EnvironmentModulesTcl, Lmod, NoModulesTool from easybuild.tools.modules import curr_module_paths, get_software_libdir, get_software_root, get_software_version from easybuild.tools.modules import invalidate_module_caches_for, modules_tool, reset_module_caches -from easybuild.tools.run import run +from easybuild.tools.run import run_shell_cmd # number of modules included for testing purposes @@ -1332,7 +1332,7 @@ def test_module_use_bash(self): self.assertIn(modules_dir, modulepath) with self.mocked_stdout_stderr(): - res = run("bash -c 'echo MODULEPATH: $MODULEPATH'") + res = run_shell_cmd("bash -c 'echo MODULEPATH: $MODULEPATH'") self.assertEqual(res.output.strip(), f"MODULEPATH: {modulepath}") self.assertIn(modules_dir, res.output) diff --git a/test/framework/run.py b/test/framework/run.py index b5faf637f4..ee9d9cb627 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -51,7 +51,7 @@ from easybuild.tools.config import update_build_option from easybuild.tools.filetools import adjust_permissions, mkdir, read_file, write_file from easybuild.tools.run import check_async_cmd, check_log_for_errors, complete_cmd, get_output_from_process -from easybuild.tools.run import RunResult, parse_log_for_error, run, run_cmd, run_cmd_qa, subprocess_terminate +from easybuild.tools.run import RunResult, parse_log_for_error, run_shell_cmd, run_cmd, run_cmd_qa, subprocess_terminate from easybuild.tools.config import ERROR, IGNORE, WARN @@ -161,11 +161,11 @@ def test_run_cmd(self): self.assertTrue(out.startswith('foo ') and out.endswith(' bar')) self.assertEqual(type(out), str) - def test_run_basic(self): - """Basic test for run function.""" + def test_run_shell_cmd_basic(self): + """Basic test for run_shell_cmd function.""" with self.mocked_stdout_stderr(): - res = run("echo hello") + res = run_shell_cmd("echo hello") self.assertEqual(res.output, "hello\n") # no reason echo hello could fail self.assertEqual(res.cmd, "echo hello") @@ -185,7 +185,7 @@ def test_run_basic(self): cmd = "cat %s" % test_file with self.mocked_stdout_stderr(): - res = run(cmd) + res = run_shell_cmd(cmd) self.assertEqual(res.cmd, cmd) self.assertEqual(res.exit_code, 0) self.assertTrue(res.output.startswith('foo ') and res.output.endswith(' bar')) @@ -241,8 +241,8 @@ def test_run_cmd_log(self): self.assertTrue(logfiles[0].startswith("easybuild")) self.assertTrue(logfiles[0].endswith("log")) - def test_run_log(self): - """Test logging of executed commands with run function.""" + def test_run_shell_cmd_log(self): + """Test logging of executed commands with run_shell_cmd function.""" fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') os.close(fd) @@ -253,7 +253,7 @@ def test_run_log(self): # command output is always logged init_logging(logfile, silent=True) with self.mocked_stdout_stderr(): - res = run("echo hello") + res = run_shell_cmd("echo hello") stop_logging(logfile) self.assertEqual(res.exit_code, 0) self.assertEqual(res.output, 'hello\n') @@ -266,7 +266,7 @@ def test_run_log(self): init_logging(logfile, silent=True) with self.mocked_stdout_stderr(): - res = run("echo hello") + res = run_shell_cmd("echo hello") stop_logging(logfile) self.assertEqual(res.exit_code, 0) self.assertEqual(res.output, 'hello\n') @@ -304,8 +304,8 @@ def handler(signum, _): signal.signal(signal.SIGALRM, orig_sigalrm_handler) signal.alarm(0) - def test_run_fail_cmd(self): - """Test run function with command that has negative exit code.""" + def test_run_shell_cmd_fail_cmd(self): + """Test run_shell_cmd function with command that has negative exit code.""" # define signal handler to call in case run takes too long def handler(signum, _): raise RuntimeError("Signal handler called with signal %s" % signum) @@ -318,7 +318,7 @@ def handler(signum, _): signal.alarm(3) with self.mocked_stdout_stderr(): - res = run("kill -9 $$", fail_on_error=False) + res = run_shell_cmd("kill -9 $$", fail_on_error=False) self.assertEqual(res.exit_code, -9) finally: @@ -335,11 +335,11 @@ def test_run_cmd_bis(self): self.assertEqual(len(out), len("hello\n" * 300)) self.assertEqual(ec, 0) - def test_run_bis(self): - """More 'complex' test for run function.""" + def test_run_shell_cmd_bis(self): + """More 'complex' test for run_shell_cmd function.""" # a more 'complex' command to run, make sure all required output is there with self.mocked_stdout_stderr(): - res = run("for j in `seq 1 3`; do for i in `seq 1 100`; do echo hello; done; sleep 1.4; done") + res = run_shell_cmd("for j in `seq 1 3`; do for i in `seq 1 100`; do echo hello; done; sleep 1.4; done") self.assertTrue(res.output.startswith('hello\nhello\n')) self.assertEqual(len(res.output), len("hello\n" * 300)) self.assertEqual(res.exit_code, 0) @@ -363,9 +363,9 @@ def test_run_cmd_work_dir(self): self.assertTrue(os.path.samefile(orig_wd, os.getcwd())) - def test_run_work_dir(self): + def test_run_shell_cmd_work_dir(self): """ - Test running command in specific directory with run function. + Test running command in specific directory with run_shell_cmd function. """ orig_wd = os.getcwd() self.assertFalse(os.path.samefile(orig_wd, self.test_prefix)) @@ -376,7 +376,7 @@ def test_run_work_dir(self): cmd = "ls | sort" with self.mocked_stdout_stderr(): - res = run(cmd, work_dir=test_dir) + res = run_shell_cmd(cmd, work_dir=test_dir) self.assertEqual(res.cmd, cmd) self.assertEqual(res.exit_code, 0) @@ -418,8 +418,8 @@ def test_run_cmd_log_output(self): self.assertTrue(out.startswith('foo ') and out.endswith(' bar')) self.assertEqual(type(out), str) - def test_run_split_stderr(self): - """Test getting split stdout/stderr output from run function.""" + def test_run_shell_cmd_split_stderr(self): + """Test getting split stdout/stderr output from run_shell_cmd function.""" cmd = ';'.join([ "echo ok", "echo warning >&2", @@ -427,7 +427,7 @@ def test_run_split_stderr(self): # by default, output contains both stdout + stderr with self.mocked_stdout_stderr(): - res = run(cmd) + res = run_shell_cmd(cmd) self.assertEqual(res.exit_code, 0) output_lines = res.output.split('\n') self.assertTrue("ok" in output_lines) @@ -435,7 +435,7 @@ def test_run_split_stderr(self): self.assertEqual(res.stderr, None) with self.mocked_stdout_stderr(): - res = run(cmd, split_stderr=True) + res = run_shell_cmd(cmd, split_stderr=True) self.assertEqual(res.exit_code, 0) self.assertEqual(res.stderr, "warning\n") self.assertEqual(res.output, "ok\n") @@ -528,8 +528,8 @@ def test_run_cmd_trace(self): self.assertEqual(stdout, '') self.assertEqual(stderr, '') - def test_run_trace(self): - """Test run in trace mode, and with tracing disabled.""" + def test_run_shell_cmd_trace(self): + """Test run_shell_cmd function in trace mode, and with tracing disabled.""" pattern = [ r"^ >> running command:", @@ -543,7 +543,7 @@ def test_run_trace(self): # trace output is enabled by default (since EasyBuild v5.0) self.mock_stdout(True) self.mock_stderr(True) - res = run("echo hello") + res = run_shell_cmd("echo hello") stdout = self.get_stdout() stderr = self.get_stderr() self.mock_stdout(False) @@ -558,7 +558,7 @@ def test_run_trace(self): self.mock_stdout(True) self.mock_stderr(True) - res = run("echo hello") + res = run_shell_cmd("echo hello") stdout = self.get_stdout() stderr = self.get_stderr() self.mock_stdout(False) @@ -576,7 +576,7 @@ def test_run_trace(self): self.mock_stdout(True) self.mock_stderr(True) - res = run("echo hello", hidden=True) + res = run_shell_cmd("echo hello", hidden=True) stdout = self.get_stdout() stderr = self.get_stderr() self.mock_stdout(False) @@ -586,8 +586,8 @@ def test_run_trace(self): self.assertEqual(stdout, '') self.assertEqual(stderr, '') - def test_run_trace_stdin(self): - """Test run under --trace + passing stdin input.""" + def test_run_shell_cmd_trace_stdin(self): + """Test run_shell_cmd function under --trace + passing stdin input.""" init_config(build_options={'trace': True}) @@ -602,7 +602,7 @@ def test_run_trace_stdin(self): self.mock_stdout(True) self.mock_stderr(True) - res = run("echo hello") + res = run_shell_cmd("echo hello") stdout = self.get_stdout() stderr = self.get_stderr() self.mock_stdout(False) @@ -616,7 +616,7 @@ def test_run_trace_stdin(self): # also test with command that is fed input via stdin self.mock_stdout(True) self.mock_stderr(True) - res = run('cat', stdin='hello') + res = run_shell_cmd('cat', stdin='hello') stdout = self.get_stdout() stderr = self.get_stderr() self.mock_stdout(False) @@ -632,7 +632,7 @@ def test_run_trace_stdin(self): # trace output can be disabled on a per-command basis by enabling 'hidden' self.mock_stdout(True) self.mock_stderr(True) - res = run("echo hello", hidden=True) + res = run_shell_cmd("echo hello", hidden=True) stdout = self.get_stdout() stderr = self.get_stderr() self.mock_stdout(False) @@ -808,17 +808,17 @@ def test_run_cmd_cache(self): run_cmd.clear_cache() - def test_run_cache(self): - """Test caching for run""" + def test_run_shell_cmd_cache(self): + """Test caching for run_shell_cmd function""" cmd = "ulimit -u" with self.mocked_stdout_stderr(): - res = run(cmd) + res = run_shell_cmd(cmd) first_out = res.output self.assertEqual(res.exit_code, 0) with self.mocked_stdout_stderr(): - res = run(cmd) + res = run_shell_cmd(cmd) cached_out = res.output self.assertEqual(res.exit_code, 0) self.assertEqual(first_out, cached_out) @@ -826,8 +826,8 @@ def test_run_cache(self): # inject value into cache to check whether executing command again really returns cached value with self.mocked_stdout_stderr(): cached_res = RunResult(cmd=cmd, output="123456", exit_code=123, stderr=None, work_dir='/test_ulimit') - run.update_cache({(cmd, None): cached_res}) - res = run(cmd) + run_shell_cmd.update_cache({(cmd, None): cached_res}) + res = run_shell_cmd(cmd) self.assertEqual(res.cmd, cmd) self.assertEqual(res.exit_code, 123) self.assertEqual(res.output, "123456") @@ -837,22 +837,22 @@ def test_run_cache(self): # also test with command that uses stdin cmd = "cat" with self.mocked_stdout_stderr(): - res = run(cmd, stdin='foo') + res = run_shell_cmd(cmd, stdin='foo') self.assertEqual(res.exit_code, 0) self.assertEqual(res.output, 'foo') # inject different output for cat with 'foo' as stdin to check whether cached value is used with self.mocked_stdout_stderr(): cached_res = RunResult(cmd=cmd, output="bar", exit_code=123, stderr=None, work_dir='/test_cat') - run.update_cache({(cmd, 'foo'): cached_res}) - res = run(cmd, stdin='foo') + run_shell_cmd.update_cache({(cmd, 'foo'): cached_res}) + res = run_shell_cmd(cmd, stdin='foo') self.assertEqual(res.cmd, cmd) self.assertEqual(res.exit_code, 123) self.assertEqual(res.output, 'bar') self.assertEqual(res.stderr, None) self.assertEqual(res.work_dir, '/test_cat') - run.clear_cache() + run_shell_cmd.clear_cache() def test_parse_log_error(self): """Test basic parse_log_for_error functionality.""" @@ -902,8 +902,8 @@ def test_run_cmd_dry_run(self): expected = """ running interactive command "some_qa_cmd"\n""" self.assertIn(expected, stdout) - def test_run_dry_run(self): - """Test use of run function under (extended) dry run.""" + def test_run_shell_cmd_dry_run(self): + """Test use of run_shell_cmd function under (extended) dry run.""" build_options = { 'extended_dry_run': True, 'silent': False, @@ -913,7 +913,7 @@ def test_run_dry_run(self): cmd = "somecommand foo 123 bar" self.mock_stdout(True) - res = run(cmd) + res = run_shell_cmd(cmd) stdout = self.get_stdout() self.mock_stdout(False) # fake output/exit code is returned for commands not actually run in dry run mode @@ -926,7 +926,7 @@ def test_run_dry_run(self): # check enabling 'hidden' self.mock_stdout(True) - res = run(cmd, hidden=True) + res = run_shell_cmd(cmd, hidden=True) stdout = self.get_stdout() self.mock_stdout(False) # fake output/exit code is returned for commands not actually run in dry run mode @@ -940,7 +940,8 @@ def test_run_dry_run(self): outfile = os.path.join(self.test_prefix, 'cmd.out') self.assertNotExists(outfile) self.mock_stdout(True) - res = run("echo 'This is always echoed' > %s; echo done; false" % outfile, fail_on_error=False, in_dry_run=True) + res = run_shell_cmd("echo 'This is always echoed' > %s; echo done; false" % outfile, + fail_on_error=False, in_dry_run=True) stdout = self.get_stdout() self.mock_stdout(False) self.assertNotIn('running command "', stdout) @@ -1228,9 +1229,9 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): ]) self.assertEqual(stdout, expected_stdout) - def test_run_with_hooks(self): + def test_run_shell_cmd_with_hooks(self): """ - Test running command with run with pre/post run_shell_cmd hooks in place. + Test running command with run_shell_cmd function with pre/post run_shell_cmd hooks in place. """ cwd = os.getcwd() @@ -1266,7 +1267,7 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): update_build_option('trace', False) with self.mocked_stdout_stderr(): - run("make") + run_shell_cmd("make") stdout = self.get_stdout() expected_stdout = '\n'.join([ diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 82f1315dc5..c86427326a 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -40,7 +40,7 @@ import easybuild.tools.systemtools as st from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import adjust_permissions, read_file, symlink, which, write_file -from easybuild.tools.run import RunResult, run +from easybuild.tools.run import RunResult, run_shell_cmd from easybuild.tools.systemtools import CPU_ARCHITECTURES, AARCH32, AARCH64, POWER, X86_64 from easybuild.tools.systemtools import CPU_FAMILIES, POWER_LE, DARWIN, LINUX, UNKNOWN from easybuild.tools.systemtools import CPU_VENDORS, AMD, APM, ARM, CAVIUM, IBM, INTEL @@ -320,8 +320,8 @@ def mocked_is_readable(mocked_fp, fp): return fp == mocked_fp -def mocked_run(cmd, **kwargs): - """Mocked version of run, with specified output for known commands.""" +def mocked_run_shell_cmd(cmd, **kwargs): + """Mocked version of run_shell_cmd, with specified output for known commands.""" known_cmds = { "gcc --version": "gcc (GCC) 5.1.1 20150618 (Red Hat 5.1.1-4)", "ldd --version": "ldd (GNU libc) 2.12; ", @@ -342,7 +342,7 @@ def mocked_run(cmd, **kwargs): if cmd in known_cmds: return RunResult(cmd=cmd, exit_code=0, output=known_cmds[cmd], stderr=None, work_dir=os.getcwd()) else: - return run(cmd, **kwargs) + return run_shell_cmd(cmd, **kwargs) def mocked_uname(): @@ -361,7 +361,7 @@ def setUp(self): self.orig_get_os_type = st.get_os_type self.orig_is_readable = st.is_readable self.orig_read_file = st.read_file - self.orig_run = st.run + self.orig_run_shell_cmd = st.run_shell_cmd self.orig_platform_dist = st.platform.dist if hasattr(st.platform, 'dist') else None self.orig_platform_uname = st.platform.uname self.orig_get_tool_version = st.get_tool_version @@ -381,7 +381,7 @@ def tearDown(self): st.get_cpu_architecture = self.orig_get_cpu_architecture st.get_os_name = self.orig_get_os_name st.get_os_type = self.orig_get_os_type - st.run = self.orig_run + st.run_shell_cmd = self.orig_run_shell_cmd if self.orig_platform_dist is not None: st.platform.dist = self.orig_platform_dist st.platform.uname = self.orig_platform_uname @@ -412,7 +412,7 @@ def test_avail_core_count_linux(self): def test_avail_core_count_darwin(self): """Test getting core count (mocked for Darwin).""" st.get_os_type = lambda: st.DARWIN - st.run = mocked_run + st.run_shell_cmd = mocked_run_shell_cmd self.assertEqual(get_avail_core_count(), 10) def test_cpu_model_native(self): @@ -450,7 +450,7 @@ def test_cpu_model_linux(self): def test_cpu_model_darwin(self): """Test getting CPU model (mocked for Darwin).""" st.get_os_type = lambda: st.DARWIN - st.run = mocked_run + st.run_shell_cmd = mocked_run_shell_cmd self.assertEqual(get_cpu_model(), "Intel(R) Core(TM) i5-4258U CPU @ 2.40GHz") def test_cpu_speed_native(self): @@ -485,7 +485,7 @@ def test_cpu_speed_linux(self): def test_cpu_speed_darwin(self): """Test getting CPU speed (mocked for Darwin).""" st.get_os_type = lambda: st.DARWIN - st.run = mocked_run + st.run_shell_cmd = mocked_run_shell_cmd self.assertEqual(get_cpu_speed(), 2400.0) def test_cpu_features_native(self): @@ -540,7 +540,7 @@ def test_cpu_features_linux(self): def test_cpu_features_darwin(self): """Test getting CPU features (mocked for Darwin).""" st.get_os_type = lambda: st.DARWIN - st.run = mocked_run + st.run_shell_cmd = mocked_run_shell_cmd expected = ['1gbpage', 'acpi', 'aes', 'apic', 'avx1.0', 'avx2', 'bmi1', 'bmi2', 'clfsh', 'cmov', 'cx16', 'cx8', 'de', 'ds', 'dscpl', 'dtes64', 'em64t', 'erms', 'est', 'f16c', 'fma', 'fpu', 'fpu_csds', 'fxsr', 'htt', 'invpcid', 'lahf', 'lzcnt', 'mca', 'mce', 'mmx', 'mon', 'movbe', 'msr', 'mtrr', @@ -634,12 +634,12 @@ def test_cpu_vendor_linux(self): def test_cpu_vendor_darwin(self): """Test getting CPU vendor (mocked for Darwin).""" st.get_os_type = lambda: st.DARWIN - st.run = mocked_run + st.run_shell_cmd = mocked_run_shell_cmd self.assertEqual(get_cpu_vendor(), INTEL) def test_cpu_family_native(self): """Test get_cpu_family function.""" - run.clear_cache() + run_shell_cmd.clear_cache() cpu_family = get_cpu_family() self.assertTrue(cpu_family in CPU_FAMILIES or cpu_family == UNKNOWN) @@ -684,8 +684,8 @@ def test_cpu_family_linux(self): def test_cpu_family_darwin(self): """Test get_cpu_family function (mocked for Darwin).""" st.get_os_type = lambda: st.DARWIN - st.run = mocked_run - run.clear_cache() + st.run_shell_cmd = mocked_run_shell_cmd + run_shell_cmd.clear_cache() self.assertEqual(get_cpu_family(), INTEL) def test_os_type(self): @@ -767,7 +767,7 @@ def test_gcc_version_native(self): def test_gcc_version_linux(self): """Test getting gcc version (mocked for Linux).""" st.get_os_type = lambda: st.LINUX - st.run = mocked_run + st.run_shell_cmd = mocked_run_shell_cmd self.assertEqual(get_gcc_version(), '5.1.1') def test_gcc_version_darwin(self): @@ -775,7 +775,7 @@ def test_gcc_version_darwin(self): st.get_os_type = lambda: st.DARWIN out = "Apple LLVM version 7.0.0 (clang-700.1.76)" mocked_run_res = RunResult(cmd="gcc --version", exit_code=0, output=out, stderr=None, work_dir=os.getcwd()) - st.run = lambda *args, **kwargs: mocked_run_res + st.run_shell_cmd = lambda *args, **kwargs: mocked_run_res self.assertEqual(get_gcc_version(), None) def test_glibc_version_native(self): @@ -786,7 +786,7 @@ def test_glibc_version_native(self): def test_glibc_version_linux(self): """Test getting glibc version (mocked for Linux).""" st.get_os_type = lambda: st.LINUX - st.run = mocked_run + st.run_shell_cmd = mocked_run_shell_cmd self.assertEqual(get_glibc_version(), '2.12') def test_glibc_version_linux_gentoo(self): @@ -817,7 +817,7 @@ def test_get_total_memory_linux(self): def test_get_total_memory_darwin(self): """Test the function that gets the total memory.""" st.get_os_type = lambda: st.DARWIN - st.run = mocked_run + st.run_shell_cmd = mocked_run_shell_cmd self.assertEqual(get_total_memory(), 8192) def test_get_total_memory_native(self): @@ -849,7 +849,7 @@ def test_det_parallelism_mocked(self): st.get_avail_core_count = lambda: 8 self.assertTrue(det_parallelism(), 8) # make 'ulimit -u' return '40', which should result in default (max) parallelism of 4 ((40-15)/6) - st.run = mocked_run + st.run_shell_cmd = mocked_run_shell_cmd self.assertTrue(det_parallelism(), 4) self.assertTrue(det_parallelism(par=6), 4) self.assertTrue(det_parallelism(maxpar=2), 2) @@ -1038,10 +1038,10 @@ def test_check_linked_shared_libs(self): os_type = get_os_type() if os_type == LINUX: with self.mocked_stdout_stderr(): - res = run("ldd %s" % bin_ls_path) + res = run_shell_cmd("ldd %s" % bin_ls_path) elif os_type == DARWIN: with self.mocked_stdout_stderr(): - res = run("otool -L %s" % bin_ls_path) + res = run_shell_cmd("otool -L %s" % bin_ls_path) else: raise EasyBuildError("Unknown OS type: %s" % os_type) From 65efc4abdf00e00f281909a50b14811aa122ac2b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 24 Aug 2023 21:16:48 +0200 Subject: [PATCH 110/312] also test with Rocky Linux 8.8 and 9.2 --- .github/workflows/end2end.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index ba6b02ea8e..7327d876ba 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -11,7 +11,8 @@ jobs: - centos-8.5 - fedora-36 - opensuse-15.4 - - rockylinux-8.7 + - rockylinux-8.8 + - rockylinux-9.2 - ubuntu-20.04 - ubuntu-22.04 fail-fast: false From 8e9f39a23a080f656b76b1d4f9e73a059ef8c19c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 25 Aug 2023 10:55:37 +0200 Subject: [PATCH 111/312] fix name of test_run_shell_cmd_fail Co-authored-by: Sam Moors --- test/framework/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/run.py b/test/framework/run.py index ee9d9cb627..4d6974bb04 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -304,7 +304,7 @@ def handler(signum, _): signal.signal(signal.SIGALRM, orig_sigalrm_handler) signal.alarm(0) - def test_run_shell_cmd_fail_cmd(self): + def test_run_shell_cmd_fail(self): """Test run_shell_cmd function with command that has negative exit code.""" # define signal handler to call in case run takes too long def handler(signum, _): From 1f16560ccdda1be3941640a880d9cf605b602c0f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 29 Aug 2023 14:06:05 +0200 Subject: [PATCH 112/312] rename run_cache decorator to run_shell_cmd_cache --- easybuild/tools/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 400146aa22..85b79a7868 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -104,10 +104,10 @@ def cache_aware_func(cmd, *args, **kwargs): return cache_aware_func -run_cache = run_cmd_cache +run_shell_cmd_cache = run_cmd_cache -@run_cache +@run_shell_cmd_cache def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, shell=True, output_file=False, stream_output=False, asynchronous=False, with_hooks=True, From ca0ce94d06c11e7b6a608173cecaac6f1dbfd1a3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 29 Aug 2023 15:11:46 +0200 Subject: [PATCH 113/312] stop running EasyBuild framework test suite with CentOS 7.9 since Python 3.6+ is required now --- .github/workflows/end2end.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 7327d876ba..dae567d575 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -7,7 +7,6 @@ jobs: strategy: matrix: container: - - centos-7.9 - centos-8.5 - fedora-36 - opensuse-15.4 From 481bf95d2e65aed546813d4b35e2d95de5acd934 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Tue, 29 Aug 2023 21:04:06 +0200 Subject: [PATCH 114/312] add definitions for ifbf and iofbf toolchain --- easybuild/toolchains/ifbf.py | 44 ++++++++++++++++++++++++++++++++ easybuild/toolchains/iofbf.py | 47 +++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 easybuild/toolchains/ifbf.py create mode 100644 easybuild/toolchains/iofbf.py diff --git a/easybuild/toolchains/ifbf.py b/easybuild/toolchains/ifbf.py new file mode 100644 index 0000000000..3507e1eab9 --- /dev/null +++ b/easybuild/toolchains/ifbf.py @@ -0,0 +1,44 @@ +## +# Copyright 2012-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for ifbf toolchain (includes Intel compilers, FlexiBLAS, and FFTW). + +Authors: + +* Sebastian Achilles (Juelich Supercomputing Centre) +""" + +from easybuild.toolchains.intel_compilers import IntelCompilersToolchain +from easybuild.toolchains.fft.fftw import Fftw +from easybuild.toolchains.linalg.flexiblas import FlexiBLAS + + +class Ifbf(IntelCompilersToolchain, FlexiBLAS, Fftw): + """ + Compiler toolchain with Intel compilers, FlexiBLAS, and FFTW + """ + NAME = 'ifbf' + SUBTOOLCHAIN = IntelCompilersToolchain.NAME + OPTIONAL = True diff --git a/easybuild/toolchains/iofbf.py b/easybuild/toolchains/iofbf.py new file mode 100644 index 0000000000..3410ffaf9f --- /dev/null +++ b/easybuild/toolchains/iofbf.py @@ -0,0 +1,47 @@ +## +# Copyright 2012-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for iofbf toolchain (includes Intel compilers, OpenMPI, +FlexiBLAS, LAPACK, ScaLAPACK and FFTW). + +Authors: + +* Sebastian Achilles (Juelich Supercomputing Centre) +""" + +from easybuild.toolchains.iompi import Iompi +from easybuild.toolchains.ifbf import Ifbf +from easybuild.toolchains.fft.fftw import Fftw +from easybuild.toolchains.linalg.flexiblas import FlexiBLAS +from easybuild.toolchains.linalg.scalapack import ScaLAPACK + + +class Iofbf(Iompi, FlexiBLAS, ScaLAPACK, Fftw): + """ + Compiler toolchain with Intel compilers (icc/ifort), OpenMPI, + FlexiBLAS, LAPACK, ScaLAPACK and FFTW. + """ + NAME = 'iofbf' + SUBTOOLCHAIN = [Iompi.NAME, Ifbf.NAME] From 98402430e2407160c44eb0b44d064c7214c1c831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Kr=C3=A1l?= Date: Wed, 30 Aug 2023 11:55:25 +0200 Subject: [PATCH 115/312] Add submodule filtering and extra configuration to git cloning commands --- easybuild/tools/filetools.py | 40 +++++++++++++++++++++++++++--------- test/framework/filetools.py | 38 +++++++++++++++++++++++----------- 2 files changed, 56 insertions(+), 22 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index a5e5f63ec6..74e2c13369 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -65,7 +65,7 @@ from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN from easybuild.tools.config import build_option, install_path from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar -from easybuild.tools.py2vs3 import HTMLParser, load_source, makedirs, std_urllib, string_type +from easybuild.tools.py2vs3 import HTMLParser, load_source, std_urllib, string_type from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars, trace_msg try: @@ -1918,9 +1918,15 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): # climb up until we hit an existing path or the empty string (for relative paths) while existing_parent_path and not os.path.exists(existing_parent_path): existing_parent_path = os.path.dirname(existing_parent_path) - makedirs(path, exist_ok=True) + os.makedirs(path) else: os.mkdir(path) + except FileExistsError as err: + if os.path.exists(path): + # This may happen if a parallel build creates the directory after we checked for its existence + _log.debug("Directory creation aborted as it seems it was already created: %s", err) + else: + raise EasyBuildError("Failed to create directory %s: %s", path, err) except OSError as err: raise EasyBuildError("Failed to create directory %s: %s", path, err) @@ -2619,6 +2625,8 @@ def get_source_tarball_from_git(filename, targetdir, git_config): recursive = git_config.pop('recursive', False) clone_into = git_config.pop('clone_into', False) keep_git_dir = git_config.pop('keep_git_dir', False) + extra_config_params = git_config.pop('extra_config_params', None) + recurse_submodules = git_config.pop('recurse_submodules', None) # input validation of git_config dict if git_config: @@ -2644,7 +2652,11 @@ def get_source_tarball_from_git(filename, targetdir, git_config): targetpath = os.path.join(targetdir, filename) # compose 'git clone' command, and run it - clone_cmd = ['git', 'clone'] + if extra_config_params: + git_cmd = 'git ' + ' '.join([f'-c {param}' for param in extra_config_params]) + else: + git_cmd = 'git' + clone_cmd = [git_cmd, 'clone'] if not keep_git_dir and not commit: # Speed up cloning by only fetching the most recent commit, not the whole history @@ -2655,6 +2667,8 @@ def get_source_tarball_from_git(filename, targetdir, git_config): clone_cmd.extend(['--branch', tag]) if recursive: clone_cmd.append('--recursive') + if recurse_submodules: + clone_cmd.extend([f"--recurse-submodules='{pat}'" for pat in recurse_submodules]) else: # checkout is done separately below for specific commits clone_cmd.append('--no-checkout') @@ -2674,16 +2688,19 @@ def get_source_tarball_from_git(filename, targetdir, git_config): # if a specific commit is asked for, check it out if commit: - checkout_cmd = ['git', 'checkout', commit] + checkout_cmd = [git_cmd, 'checkout', commit] if recursive: - checkout_cmd.extend(['&&', 'git', 'submodule', 'update', '--init', '--recursive']) + checkout_cmd.extend(['&&', git_cmd, 'submodule', 'update', '--init', '--recursive']) + elif recurse_submodules: + checkout_cmd.extend(['&&', git_cmd, 'submodule', 'update', '--init']) + checkout_cmd.extend([f"--recurse-submodules='{pat}'" for pat in recurse_submodules]) run.run_cmd(' '.join(checkout_cmd), log_all=True, simple=True, regexp=False, path=repo_name) elif not build_option('extended_dry_run'): # If we wanted to get a tag make sure we actually got a tag and not a branch with the same name # This doesn't make sense in dry-run mode as we don't have anything to check - cmd = 'git describe --exact-match --tags HEAD' + cmd = f'{git_cmd} describe --exact-match --tags HEAD' # Note: Disable logging to also disable the error handling in run_cmd (out, ec) = run.run_cmd(cmd, log_ok=False, log_all=False, regexp=False, path=repo_name) if ec != 0 or tag not in out.splitlines(): @@ -2696,13 +2713,16 @@ def get_source_tarball_from_git(filename, targetdir, git_config): # make the repo unshallow first; # this is equivalent with 'git fetch -unshallow' in Git 1.8.3+ # (first fetch seems to do nothing, unclear why) - cmds.append('git fetch --depth=2147483647 && git fetch --depth=2147483647') + cmds.append(f'{git_cmd} fetch --depth=2147483647 && git fetch --depth=2147483647') - cmds.append('git checkout refs/tags/' + tag) + cmds.append(f'{git_cmd} checkout refs/tags/' + tag) # Clean all untracked files, e.g. from left-over submodules - cmds.append('git clean --force -d -x') + cmds.append(f'{git_cmd} clean --force -d -x') if recursive: - cmds.append('git submodule update --init --recursive') + cmds.append(f'{git_cmd} submodule update --init --recursive') + elif recurse_submodules: + cmds.append(f'{git_cmd} submodule update --init ') + cmds[-1] += ' '.join([f"--recurse-submodules='{pat}'" for pat in recurse_submodules]) for cmd in cmds: run.run_cmd(cmd, log_all=True, simple=True, regexp=False, path=repo_name) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index b87b9359b0..d8215ce979 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -45,7 +45,6 @@ from unittest import TextTestRunner from easybuild.tools import run import easybuild.tools.filetools as ft -import easybuild.tools.py2vs3 as py2vs3 from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import IGNORE, ERROR, build_option, update_build_option from easybuild.tools.multidiff import multidiff @@ -2809,6 +2808,32 @@ def run_check(): ]) % git_repo run_check() + git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite'] + expected = '\n'.join([ + ' running command "git clone --depth 1 --branch tag_for_tests --recursive' + + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"', + r" \(in .*/tmp.*\)", + r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r" \(in .*/tmp.*\)", + ]) % git_repo + run_check() + + git_config['extra_config_params'] = [ + 'submodule."fastahack".active=false', + 'submodule."sha1".active=false', + ] + expected = '\n'.join([ + ' running command "git -c submodule."fastahack".active=false -c submodule."sha1".active=false' + + ' clone --depth 1 --branch tag_for_tests --recursive' + + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"', + r" \(in .*/tmp.*\)", + r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r" \(in .*/tmp.*\)", + ]) % git_repo + run_check() + del git_config['recurse_submodules'] + del git_config['extra_config_params'] + git_config['keep_git_dir'] = True expected = '\n'.join([ r' running command "git clone --branch tag_for_tests --recursive %(git_repo)s"', @@ -3390,17 +3415,6 @@ def test_set_gid_sticky_bits(self): self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) - def test_compat_makedirs(self): - """Test compatibility layer for Python3 os.makedirs""" - name = os.path.join(self.test_prefix, 'folder') - self.assertNotExists(name) - py2vs3.makedirs(name) - self.assertExists(name) - # exception is raised because file exists (OSError in Python 2, FileExistsError in Python 3) - self.assertErrorRegex(Exception, '.*', py2vs3.makedirs, name) - py2vs3.makedirs(name, exist_ok=True) # No error - self.assertExists(name) - def test_create_unused_dir(self): """Test create_unused_dir function.""" path = ft.create_unused_dir(self.test_prefix, 'folder') From 012fa749c9e7b4401040715aa3de64c7b1d41d12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Kr=C3=A1l?= Date: Wed, 30 Aug 2023 12:00:28 +0200 Subject: [PATCH 116/312] fix conflicts --- easybuild/tools/filetools.py | 10 ++-------- test/framework/filetools.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 74e2c13369..d6b1c5cf90 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -65,7 +65,7 @@ from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN from easybuild.tools.config import build_option, install_path from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar -from easybuild.tools.py2vs3 import HTMLParser, load_source, std_urllib, string_type +from easybuild.tools.py2vs3 import HTMLParser, load_source, makedirs, std_urllib, string_type from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars, trace_msg try: @@ -1918,15 +1918,9 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): # climb up until we hit an existing path or the empty string (for relative paths) while existing_parent_path and not os.path.exists(existing_parent_path): existing_parent_path = os.path.dirname(existing_parent_path) - os.makedirs(path) + makedirs(path, exist_ok=True) else: os.mkdir(path) - except FileExistsError as err: - if os.path.exists(path): - # This may happen if a parallel build creates the directory after we checked for its existence - _log.debug("Directory creation aborted as it seems it was already created: %s", err) - else: - raise EasyBuildError("Failed to create directory %s: %s", path, err) except OSError as err: raise EasyBuildError("Failed to create directory %s: %s", path, err) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index d8215ce979..3921d422e7 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -45,6 +45,7 @@ from unittest import TextTestRunner from easybuild.tools import run import easybuild.tools.filetools as ft +import easybuild.tools.py2vs3 as py2vs3 from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import IGNORE, ERROR, build_option, update_build_option from easybuild.tools.multidiff import multidiff @@ -3415,6 +3416,17 @@ def test_set_gid_sticky_bits(self): self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + def test_compat_makedirs(self): + """Test compatibility layer for Python3 os.makedirs""" + name = os.path.join(self.test_prefix, 'folder') + self.assertNotExists(name) + py2vs3.makedirs(name) + self.assertExists(name) + # exception is raised because file exists (OSError in Python 2, FileExistsError in Python 3) + self.assertErrorRegex(Exception, '.*', py2vs3.makedirs, name) + py2vs3.makedirs(name, exist_ok=True) # No error + self.assertExists(name) + def test_create_unused_dir(self): """Test create_unused_dir function.""" path = ft.create_unused_dir(self.test_prefix, 'folder') From b6f3e7278ef79ebd0bdb7d19a10224af5a10988a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Kr=C3=A1l?= Date: Wed, 30 Aug 2023 12:52:59 +0200 Subject: [PATCH 117/312] fix string formating syntax --- easybuild/tools/filetools.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index d6b1c5cf90..812b52a420 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2647,7 +2647,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): # compose 'git clone' command, and run it if extra_config_params: - git_cmd = 'git ' + ' '.join([f'-c {param}' for param in extra_config_params]) + git_cmd = 'git ' + ' '.join(['-c %s' % param for param in extra_config_params]) else: git_cmd = 'git' clone_cmd = [git_cmd, 'clone'] @@ -2662,7 +2662,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): if recursive: clone_cmd.append('--recursive') if recurse_submodules: - clone_cmd.extend([f"--recurse-submodules='{pat}'" for pat in recurse_submodules]) + clone_cmd.extend(["--recurse-submodules='%s'" % pat for pat in recurse_submodules]) else: # checkout is done separately below for specific commits clone_cmd.append('--no-checkout') @@ -2687,14 +2687,14 @@ def get_source_tarball_from_git(filename, targetdir, git_config): checkout_cmd.extend(['&&', git_cmd, 'submodule', 'update', '--init', '--recursive']) elif recurse_submodules: checkout_cmd.extend(['&&', git_cmd, 'submodule', 'update', '--init']) - checkout_cmd.extend([f"--recurse-submodules='{pat}'" for pat in recurse_submodules]) + checkout_cmd.extend(["--recurse-submodules='%s'" % pat for pat in recurse_submodules]) run.run_cmd(' '.join(checkout_cmd), log_all=True, simple=True, regexp=False, path=repo_name) elif not build_option('extended_dry_run'): # If we wanted to get a tag make sure we actually got a tag and not a branch with the same name # This doesn't make sense in dry-run mode as we don't have anything to check - cmd = f'{git_cmd} describe --exact-match --tags HEAD' + cmd = '%s describe --exact-match --tags HEAD' % git_cmd # Note: Disable logging to also disable the error handling in run_cmd (out, ec) = run.run_cmd(cmd, log_ok=False, log_all=False, regexp=False, path=repo_name) if ec != 0 or tag not in out.splitlines(): @@ -2707,16 +2707,16 @@ def get_source_tarball_from_git(filename, targetdir, git_config): # make the repo unshallow first; # this is equivalent with 'git fetch -unshallow' in Git 1.8.3+ # (first fetch seems to do nothing, unclear why) - cmds.append(f'{git_cmd} fetch --depth=2147483647 && git fetch --depth=2147483647') + cmds.append('%s fetch --depth=2147483647 && git fetch --depth=2147483647' % git_cmd) - cmds.append(f'{git_cmd} checkout refs/tags/' + tag) + cmds.append('%s checkout refs/tags/' % git_cmd + tag) # Clean all untracked files, e.g. from left-over submodules - cmds.append(f'{git_cmd} clean --force -d -x') + cmds.append('%s clean --force -d -x' % git_cmd) if recursive: - cmds.append(f'{git_cmd} submodule update --init --recursive') + cmds.append('%s submodule update --init --recursive' % git_cmd) elif recurse_submodules: - cmds.append(f'{git_cmd} submodule update --init ') - cmds[-1] += ' '.join([f"--recurse-submodules='{pat}'" for pat in recurse_submodules]) + cmds.append('%s submodule update --init ' % git_cmd) + cmds[-1] += ' '.join(["--recurse-submodules='%s'" % pat for pat in recurse_submodules]) for cmd in cmds: run.run_cmd(cmd, log_all=True, simple=True, regexp=False, path=repo_name) From ad3d411e80adb91dfa2bd754aca1dc5eb1f2b80b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 30 Aug 2023 14:40:28 +0200 Subject: [PATCH 118/312] improve docstring for validate_github_token to cover both classic and fine-grained GitHub tokens --- easybuild/tools/github.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 3bb21a6b95..6feb86284d 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -2243,7 +2243,8 @@ def install_github_token(github_user, silent=False): def validate_github_token(token, github_user): """ Check GitHub token: - * see if it conforms expectations (character classes depending on type, length of 40-93), + * see if it conforms expectations (classic GitHub token with only [0-9a-f] characters and length of 40 starting with 'ghp_'), + or fine-grained GitHub token with only alphanumeric ([a-zA-Z0-9]) + '_' and length of 93 starting with 'github_pat_'), * see if it can be used for authenticated access. """ # cfr. https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/ From 4ab0b76e656450ef2ed7ee5001e9aae867dfb9af Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 30 Aug 2023 14:42:52 +0200 Subject: [PATCH 119/312] fix excessively long lines in docstring for validate_github_token --- easybuild/tools/github.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 6feb86284d..d6093a9df3 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -2243,8 +2243,9 @@ def install_github_token(github_user, silent=False): def validate_github_token(token, github_user): """ Check GitHub token: - * see if it conforms expectations (classic GitHub token with only [0-9a-f] characters and length of 40 starting with 'ghp_'), - or fine-grained GitHub token with only alphanumeric ([a-zA-Z0-9]) + '_' and length of 93 starting with 'github_pat_'), + * see if it conforms expectations (classic GitHub token with only [0-9a-f] characters + and length of 40 starting with 'ghp_', or fine-grained GitHub token with only + alphanumeric ([a-zA-Z0-9]) characters + '_' and length of 93 starting with 'github_pat_'), * see if it can be used for authenticated access. """ # cfr. https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/ From 3e94aafd744851847c976cd145bfbde4a4d24e0a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 30 Aug 2023 14:59:30 +0200 Subject: [PATCH 120/312] enhance test for validate_github_token so testing with fine-grained GitHub token can also be done --- test/framework/github.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/framework/github.py b/test/framework/github.py index 9df53736e4..fb0d59cc49 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -583,6 +583,11 @@ def test_validate_github_token(self): if token_old_format: self.assertTrue(gh.validate_github_token(token_old_format, GITHUB_TEST_ACCOUNT)) + # if a fine-grained token is available, test with that too + finegrained_token = os.getenv('TEST_GITHUB_TOKEN_FINEGRAINED') + if finegrained_token: + self.assertTrue(gh.validate_github_token(finegrained_token, GITHUB_TEST_ACCOUNT)) + def test_github_find_easybuild_easyconfig(self): """Test for find_easybuild_easyconfig function""" if self.skip_github_tests: From 4c36bb21e016b5d4f437407f3d056fb093411252 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Kr=C3=A1l?= Date: Wed, 30 Aug 2023 16:48:46 +0200 Subject: [PATCH 121/312] Fix typo in submodule filtering of --- easybuild/tools/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 812b52a420..26ebb0782e 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2685,7 +2685,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): checkout_cmd = [git_cmd, 'checkout', commit] if recursive: checkout_cmd.extend(['&&', git_cmd, 'submodule', 'update', '--init', '--recursive']) - elif recurse_submodules: + if recurse_submodules: checkout_cmd.extend(['&&', git_cmd, 'submodule', 'update', '--init']) checkout_cmd.extend(["--recurse-submodules='%s'" % pat for pat in recurse_submodules]) From 2e94335c55e17514098fda26a52bb8badcfaf661 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Kr=C3=A1l?= Date: Wed, 30 Aug 2023 17:19:40 +0200 Subject: [PATCH 122/312] fix submodule filtering bug, add test --- easybuild/tools/filetools.py | 10 ++++++---- test/framework/filetools.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 26ebb0782e..3a99978db9 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2683,11 +2683,13 @@ def get_source_tarball_from_git(filename, targetdir, git_config): # if a specific commit is asked for, check it out if commit: checkout_cmd = [git_cmd, 'checkout', commit] - if recursive: - checkout_cmd.extend(['&&', git_cmd, 'submodule', 'update', '--init', '--recursive']) - if recurse_submodules: + + if recursive or recurse_submodules: checkout_cmd.extend(['&&', git_cmd, 'submodule', 'update', '--init']) - checkout_cmd.extend(["--recurse-submodules='%s'" % pat for pat in recurse_submodules]) + if recursive: + checkout_cmd.append('--recursive') + if recurse_submodules: + checkout_cmd.extend(["--recurse-submodules='%s'" % pat for pat in recurse_submodules]) run.run_cmd(' '.join(checkout_cmd), log_all=True, simple=True, regexp=False, path=repo_name) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 3921d422e7..80a23d83cc 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2857,7 +2857,20 @@ def run_check(): ]) % git_repo run_check() + git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite'] + expected = '\n'.join([ + r' running command "git clone --no-checkout %(git_repo)s"', + r" \(in .*/tmp.*\)", + ' running command "git checkout 8456f86 && git submodule update --init --recursive' + + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\'"', + r" \(in testrepository\)", + r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r" \(in .*/tmp.*\)", + ]) % git_repo + run_check() + del git_config['recursive'] + del git_config['recurse_submodules'] expected = '\n'.join([ r' running command "git clone --no-checkout %(git_repo)s"', r" \(in .*/tmp.*\)", From f725ac265c45fe015b19b894a561a4613994d793 Mon Sep 17 00:00:00 2001 From: Ward Poelmans Date: Wed, 16 Aug 2023 17:32:58 +0200 Subject: [PATCH 123/312] Only add extensions in module file if there are extensions This fixes #4330. --- easybuild/tools/module_generator.py | 3 ++- test/framework/module_generator.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index aaf97d3194..72afdd0896 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -622,7 +622,8 @@ def _generate_extensions_list(self): """ Generate a list of all extensions in name/version format """ - return self.app.make_extension_string(name_version_sep='/', ext_sep=',').split(',') + exts_str = self.app.make_extension_string(name_version_sep='/', ext_sep=',') + return exts_str.split(',') if exts_str else [] def _generate_help_text(self): """ diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index b6bec17093..a0c6131140 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -756,6 +756,16 @@ def test_module_extensions(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(desc), "Pattern '%s' found in: %s" % (regex.pattern, desc)) + # check if the extensions is missing if there are no extensions + test_ec = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-test.eb') + + ec = EasyConfig(test_ec) + eb = EasyBlock(ec) + modgen = self.MODULE_GENERATOR_CLASS(eb) + desc = modgen.get_description() + + self.assertFalse(re.search(r"\s*extensions\(", desc), "No extensions found in: %s" % desc) + def test_prepend_paths(self): """Test generating prepend-paths statements.""" # test prepend_paths From 0981458eb701b24f06211dbea5bd60df29923853 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 9 Sep 2023 18:16:10 +0200 Subject: [PATCH 124/312] enhance test_checksum_step to check lack of error message in log when checksums.json is not found --- test/framework/easyblock.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 4f27e86fb2..6afc9db94f 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -39,6 +39,7 @@ from unittest import TextTestRunner import easybuild.tools.systemtools as st +from easybuild.base import fancylogger from easybuild.framework.easyblock import EasyBlock, get_easyblock_instance from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.easyconfig.easyconfig import EasyConfig @@ -2424,6 +2425,30 @@ def test_checksum_step(self): eb.fetch_sources() eb.checksum_step() + with self.mocked_stdout_stderr() as (stdout, stderr): + + # using checksum-less test easyconfig in location that does not provide checksums.json + test_ec = os.path.join(self.test_prefix, 'test-no-checksums.eb') + copy_file(toy_ec, test_ec) + write_file(test_ec, 'checksums = []', append=True) + ec = process_easyconfig(test_ec)[0] + + # enable logging to screen, so we can check whether error is logged when checksums.json is not found + fancylogger.logToScreen(enable=True, stdout=True) + + eb = get_easyblock_instance(ec) + eb.fetch_sources() + eb.checksum_step() + + fancylogger.logToScreen(enable=False, stdout=True) + stdout = self.get_stdout() + + # make sure there's no error logged for not finding checksums.json, + # see also https://github.com/easybuilders/easybuild-framework/issues/4301 + regex = re.compile("ERROR .*Couldn't find file checksums.json anywhere", re.M) + regex.search(stdout) + self.assertFalse(regex.search(stdout), "Pattern '%s' should not be found in log" % regex.pattern) + # fiddle with checksum to check whether faulty checksum is catched copy_file(toy_ec, self.test_prefix) toy_ec = os.path.join(self.test_prefix, os.path.basename(toy_ec)) From f60d2c6bd01e45facf01305d53444333194512f9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 9 Sep 2023 19:07:41 +0200 Subject: [PATCH 125/312] improve error message produced by verify_checksum --- easybuild/tools/filetools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index e66a532c10..6e4bd2d83e 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1298,8 +1298,8 @@ def verify_checksum(path, checksums): # no matching checksums return False else: - raise EasyBuildError("Invalid checksum spec '%s', should be a string (MD5 or SHA256) " - "2-tuple (type, value) or tuple of alternative checksum specs.", + raise EasyBuildError("Invalid checksum spec '%s': should be a string (MD5 or SHA256), " + "2-tuple (type, value), or tuple of alternative checksum specs.", checksum) actual_checksum = compute_checksum(path, typ) From 22273a2df0503fb466184bbfb95bad4523bc9d69 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Sat, 9 Sep 2023 22:53:43 +0200 Subject: [PATCH 126/312] prepare release notes for EasyBuild v4.8.1 + bump version to 4.8.1 --- RELEASE_NOTES | 30 ++++++++++++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 8f5ec0f1e0..5cd325dd66 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,36 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.8.1 (11 September 2023) +-------------------------- + +update/bugfix release + +- various enhancements, including: + - add end-to-end test for running EasyBuild in different Linux distros using containers (#3968) + - suggest default title in `--review-pr` (#4287) + - add `build_and_install_loop` hooks to run before and after the install loop for individual easyconfigs (#4304) + - Implement hooks for failure scenarios: `crash_hook`, `cancel_hook`, `fail_hook` (#4315) + - add postiter hook to the list of steps so the corresponding hook can be used (#4316) + - add `run_shell_cmd` hook (#4323) + - add `build_info_msg` easyconfig parameter to print message during installation of an easyconfig (#4324) + - add `--silence-hook-trigger` configuration option to supress printing of debug message every time a hook is triggered (#4329) + - add support for using fine grained Github tokens (#4332) + - add definitions for ifbf and iofbf toolchain (#4337) + - add support for submodule filtering and specifying extra Git configuration in `git_config` (#4338) +- various bug fixes, including: + - Improve error when checksum dict has no entry for a file (#4150) + - don't fail in `mkdir` if path gets created while processing it (#4300) + - Ignore request for external module (meta)data when no modules tool is active (#4308) + - use sys.executable to obtain path to `python` command, rather than assuming that `python` command is available in `$PATH` (#4309) + - fix `test_add_and_remove_module_path` by replacing string comparison of paths by checking whether they point to the same path (since symlinks may cause trouble) (#4312) + - enhance `Toolchain.get_flag` to handle lists (#4319) + - remove crash hook (since it doesn't work) (#4325) + - don't use `FileExistsError` in `mkdir` function (doesn't exist in Python 2.7) (#4328) + - only add extensions in module file if there are extensions (#4331) + - Fix submodule filtering bug in `git_config` (#4339) + + v4.8.0 (7 July 2023) -------------------- diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 61f8473a0e..e0016710ae 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -45,7 +45,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.8.1.dev0') +VERSION = LooseVersion('4.8.1') UNKNOWN = 'UNKNOWN' From 6e05649c242e5bf0750885e7b924d4debdd2d74c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 10 Sep 2023 10:57:26 +0200 Subject: [PATCH 127/312] minor tweaks to 4.8.1 release notes --- RELEASE_NOTES | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 5cd325dd66..5bd07d4306 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -13,25 +13,23 @@ update/bugfix release - add end-to-end test for running EasyBuild in different Linux distros using containers (#3968) - suggest default title in `--review-pr` (#4287) - add `build_and_install_loop` hooks to run before and after the install loop for individual easyconfigs (#4304) - - Implement hooks for failure scenarios: `crash_hook`, `cancel_hook`, `fail_hook` (#4315) + - implement support for `cancel_hook` and `fail_hook` (#4315, #4325) - add postiter hook to the list of steps so the corresponding hook can be used (#4316) - add `run_shell_cmd` hook (#4323) - add `build_info_msg` easyconfig parameter to print message during installation of an easyconfig (#4324) - add `--silence-hook-trigger` configuration option to supress printing of debug message every time a hook is triggered (#4329) - add support for using fine grained Github tokens (#4332) - add definitions for ifbf and iofbf toolchain (#4337) - - add support for submodule filtering and specifying extra Git configuration in `git_config` (#4338) + - add support for submodule filtering and specifying extra Git configuration in `git_config` (#4338, #4339) - various bug fixes, including: - - Improve error when checksum dict has no entry for a file (#4150) - - don't fail in `mkdir` if path gets created while processing it (#4300) - - Ignore request for external module (meta)data when no modules tool is active (#4308) - - use sys.executable to obtain path to `python` command, rather than assuming that `python` command is available in `$PATH` (#4309) + - improve error when checksum dict has no entry for a file (#4150) + - avoid error being logged when `checksums.json` is not found (#4261) + - don't fail in `mkdir` if path gets created while processing it (#4300, #4328) + - ignore request for external module (meta)data when no modules tool is active (#4308) + - use sys.executable to obtain path to `python` command in tests, rather than assuming that `python` command is available in `$PATH` (#4309) - fix `test_add_and_remove_module_path` by replacing string comparison of paths by checking whether they point to the same path (since symlinks may cause trouble) (#4312) - enhance `Toolchain.get_flag` to handle lists (#4319) - - remove crash hook (since it doesn't work) (#4325) - - don't use `FileExistsError` in `mkdir` function (doesn't exist in Python 2.7) (#4328) - only add extensions in module file if there are extensions (#4331) - - Fix submodule filtering bug in `git_config` (#4339) v4.8.0 (7 July 2023) From 5e834e96de31a39e2775235aac7eca1db6b06a67 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Mon, 11 Sep 2023 12:57:49 +0100 Subject: [PATCH 128/312] Supress ==> Suppress --- easybuild/tools/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index b60e57513d..a55cea4bbf 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -501,7 +501,7 @@ def override_options(self): 'silence-deprecation-warnings': ( "Silence specified deprecation warnings out of (%s)" % ', '.join(all_deprecations), 'strlist', 'extend', []), - 'silence-hook-trigger': ("Supress printing of debug message every time a hook is triggered", + 'silence-hook-trigger': ("Suppress printing of debug message every time a hook is triggered", None, 'store_true', False), 'skip-extensions': ("Skip installation of extensions", None, 'store_true', False), 'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'), From 1464fe029a9919968baa13a3d9c7b110c669974a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 11 Sep 2023 16:52:19 +0200 Subject: [PATCH 129/312] bump version to 4.8.2dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index e0016710ae..e34584df36 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -45,7 +45,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.8.1') +VERSION = LooseVersion('4.8.2.dev0') UNKNOWN = 'UNKNOWN' From 299a4511b65a462187e3cd6f55d24a24488a2461 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 11 Sep 2023 20:50:29 +0200 Subject: [PATCH 130/312] clean up stray regex.search in test_checksum_step --- test/framework/easyblock.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 6afc9db94f..d526479f52 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -2446,7 +2446,6 @@ def test_checksum_step(self): # make sure there's no error logged for not finding checksums.json, # see also https://github.com/easybuilders/easybuild-framework/issues/4301 regex = re.compile("ERROR .*Couldn't find file checksums.json anywhere", re.M) - regex.search(stdout) self.assertFalse(regex.search(stdout), "Pattern '%s' should not be found in log" % regex.pattern) # fiddle with checksum to check whether faulty checksum is catched From c3fc220401ec0dbc34b5ac531d226ffa01a54e42 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Sep 2023 11:34:51 +0200 Subject: [PATCH 131/312] skip 4 broken tests when using Python 2, so other 850 tests can be run with Python 2 --- test/framework/easyblock.py | 7 +++++++ test/framework/toy_build.py | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index d526479f52..819d74fc7a 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1510,6 +1510,13 @@ def test_fetch_sources(self): def test_download_instructions(self): """Test use of download_instructions easyconfig parameter.""" + + # skip test when using Python 2, since it somehow fails then, + # cfr. https://github.com/easybuilders/easybuild-framework/pull/4333 + if sys.version_info[0] == 2: + print("Skipping test_download_instructions because Python 2.x is being used") + return + orig_test_ec = '\n'.join([ "easyblock = 'ConfigureMake'", "name = 'software_with_missing_sources'", diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index fb8ec907dc..757c73f30a 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2109,6 +2109,13 @@ def test_package_skip(self): def test_regtest(self): """Test use of --regtest.""" + + # skip test when using Python 2, since it somehow fails then, + # cfr. https://github.com/easybuilders/easybuild-framework/pull/4333 + if sys.version_info[0] == 2: + print("Skipping test_regtest because Python 2.x is being used") + return + self.test_toy_build(extra_args=['--regtest', '--sequential'], verify=False) # just check whether module exists @@ -2125,6 +2132,12 @@ def test_minimal_toolchains(self): def test_reproducibility(self): """Test toy build produces expected reproducibility files""" + # skip test when using Python 2, since it somehow fails then, + # cfr. https://github.com/easybuilders/easybuild-framework/pull/4333 + if sys.version_info[0] == 2: + print("Skipping test_reproducibility because Python 2.x is being used") + return + # We need hooks for a complete test hooks_filename = 'my_hooks.py' hooks_file = os.path.join(self.test_prefix, hooks_filename) @@ -3594,6 +3607,12 @@ def __exit__(self, type, value, traceback): def test_toy_lock_cleanup_signals(self): """Test cleanup of locks after EasyBuild session gets a cancellation signal.""" + # skip test when using Python 2, since it somehow fails then, + # cfr. https://github.com/easybuilders/easybuild-framework/pull/4333 + if sys.version_info[0] == 2: + print("Skipping test_toy_lock_cleanup_signals because Python 2.x is being used") + return + orig_wd = os.getcwd() locks_dir = os.path.join(self.test_installpath, 'software', '.locks') From 40d81a0ccf1c0e9aac1f0f4ff6e12d8d40e1d03b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Sep 2023 14:29:36 +0200 Subject: [PATCH 132/312] relax number of expected 'waiting' messages in test_toy_build_lock --- test/framework/toy_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 757c73f30a..9ccb3c08df 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3562,7 +3562,7 @@ def __exit__(self, type, value, traceback): wait_matches = wait_regex.findall(stdout) # we can't rely on an exact number of 'waiting' messages, so let's go with a range... - self.assertIn(len(wait_matches), range(2, 5)) + self.assertIn(len(wait_matches), range(1, 5)) self.assertTrue(ok_regex.search(stdout), "Pattern '%s' found in: %s" % (ok_regex.pattern, stdout)) From e759c92de2a5804930977df124910af6b8f883ed Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 19 Sep 2023 12:28:02 +0200 Subject: [PATCH 133/312] add documentation for major version templates E.g. `%(cudamajver)s` (introduced by #2850) is available but not documented. --- easybuild/framework/easyconfig/templates.py | 5 +++-- easybuild/tools/docs.py | 5 +++++ test/framework/docs.py | 1 + test/framework/easyconfig.py | 2 +- test/framework/options.py | 2 ++ 5 files changed, 12 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index d0a209c192..465028c3ca 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -77,9 +77,9 @@ ('installdir', "Installation directory"), ('start_dir', "Directory in which the build process begins"), ] -# software names for which to define ver and shortver templates +# software names for which to define ver, majver and shortver templates TEMPLATE_SOFTWARE_VERSIONS = [ - # software name, prefix for *ver and *shortver + # software name, prefix for *ver, *majver and *shortver ('CUDA', 'cuda'), ('CUDAcore', 'cuda'), ('Java', 'java'), @@ -427,6 +427,7 @@ def template_documentation(): # step 2: add *ver/*shortver templates for software listed in TEMPLATE_SOFTWARE_VERSIONS doc.append("Template names/values for (short) software versions") for name, pref in TEMPLATE_SOFTWARE_VERSIONS: + doc.append("%s%%(%smajver)s: major version for %s" % (indent_l1, pref, name)) doc.append("%s%%(%sshortver)s: short version for %s (.)" % (indent_l1, pref, name)) doc.append("%s%%(%sver)s: full version for %s" % (indent_l1, pref, name)) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 1fcc6ece83..4f4bfca99c 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -440,6 +440,7 @@ def avail_easyconfig_templates_txt(): # step 2: add SOFTWARE_VERSIONS doc.append('Template names/values for (short) software versions') for name, pref in TEMPLATE_SOFTWARE_VERSIONS: + doc.append("%s%%(%smajver)s: major version for %s" % (INDENT_4SPACES, pref, name)) doc.append("%s%%(%sshortver)s: short version for %s (.)" % (INDENT_4SPACES, pref, name)) doc.append("%s%%(%sver)s: full version for %s" % (INDENT_4SPACES, pref, name)) doc.append('') @@ -494,8 +495,10 @@ def avail_easyconfig_templates_rst(): ver = [] ver_desc = [] for name, pref in TEMPLATE_SOFTWARE_VERSIONS: + ver.append('``%%(%smajver)s``' % pref) ver.append('``%%(%sshortver)s``' % pref) ver.append('``%%(%sver)s``' % pref) + ver_desc.append('major version for %s' % name) ver_desc.append('short version for %s (.)' % name) ver_desc.append('full version for %s' % name) table_values = [ver, ver_desc] @@ -557,8 +560,10 @@ def avail_easyconfig_templates_md(): ver = [] ver_desc = [] for name, pref in TEMPLATE_SOFTWARE_VERSIONS: + ver.append('``%%(%smajver)s``' % pref) ver.append('``%%(%sshortver)s``' % pref) ver.append('``%%(%sver)s``' % pref) + ver_desc.append('major version for %s' % name) ver_desc.append('short version for %s (``.``)' % name) ver_desc.append('full version for %s' % name) table_values = [ver, ver_desc] diff --git a/test/framework/docs.py b/test/framework/docs.py index 146d05b24c..84d862bd3c 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -785,6 +785,7 @@ def test_avail_easyconfig_templates(self): r"^Template names/values derived from easyconfig instance", r"^\s+%\(version_major\)s: Major version", r"^Template names/values for \(short\) software versions", + r"^\s+%\(pymajver\)s: major version for Python", r"^\s+%\(pyshortver\)s: short version for Python \(\.\)", r"^Template constants that can be used in easyconfigs", r"^\s+SOURCE_TAR_GZ: Source \.tar\.gz bundle \(%\(name\)s-%\(version\)s.tar.gz\)", diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index ab1924088d..14ed69084b 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1290,7 +1290,7 @@ def test_templating_doc(self): # expected length: 1 per constant and 2 extra per constantgroup (title + empty line in between) temps = [ easyconfig.templates.TEMPLATE_NAMES_EASYCONFIG, - easyconfig.templates.TEMPLATE_SOFTWARE_VERSIONS * 2, + easyconfig.templates.TEMPLATE_SOFTWARE_VERSIONS * 3, easyconfig.templates.TEMPLATE_NAMES_CONFIG, easyconfig.templates.TEMPLATE_NAMES_LOWER, easyconfig.templates.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, diff --git a/test/framework/options.py b/test/framework/options.py index f4d331519a..51b67e39e5 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -565,6 +565,7 @@ def run_test(fmt=None): pattern_lines = [ r'^``%\(version_major\)s``\s+Major version\s*$', r'^``%\(cudaver\)s``\s+full version for CUDA\s*$', + r'^``%\(cudamajver\)s``\s+major version for CUDA\s*$', r'^``%\(pyshortver\)s``\s+short version for Python \(.\)\s*$', r'^\* ``%\(name\)s``$', r'^``%\(namelower\)s``\s+lower case of value of name\s*$', @@ -576,6 +577,7 @@ def run_test(fmt=None): pattern_lines = [ r'^\s+%\(version_major\)s: Major version$', r'^\s+%\(cudaver\)s: full version for CUDA$', + r'^\s+%\(cudamajver\)s: major version for CUDA$', r'^\s+%\(pyshortver\)s: short version for Python \(.\)$', r'^\s+%\(name\)s$', r'^\s+%\(namelower\)s: lower case of value of name$', From 5628b280ef144da1248f89aae019d22f09ff7ec6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 27 Sep 2023 08:49:04 +0200 Subject: [PATCH 134/312] reset tempdir to avoid that tmpdir path gets progressively deeper with each easystack item (fixes #4291) --- easybuild/main.py | 4 +++- test/framework/easystack.py | 30 ++++++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 1a9aa736a7..73550e3998 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -42,6 +42,7 @@ import os import stat import sys +import tempfile import traceback # IMPORTANT this has to be the first easybuild import as it customises the logging @@ -254,8 +255,9 @@ def process_easystack(easystack_path, args, logfile, testing, init_session_state easyconfig._easyconfigs_cache.clear() easyconfig._easyconfig_files_cache.clear() - # restore environment + # restore environment and reset tempdir (to avoid tmpdir path getting progressively longer) restore_env(init_env) + tempfile.tempdir = None # If EasyConfig specific arguments were supplied in EasyStack file # merge arguments with original command line args diff --git a/test/framework/easystack.py b/test/framework/easystack.py index ac36d7a26f..6951857129 100644 --- a/test/framework/easystack.py +++ b/test/framework/easystack.py @@ -31,6 +31,7 @@ import os import re import sys +import tempfile from unittest import TextTestRunner import easybuild.tools.build_log @@ -129,11 +130,22 @@ def test_easystack_invalid_key2(self): self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) def test_easystack_restore_env_after_each_build(self): - """Test that the build environment is reset for each easystack item""" + """Test that the build environment and tmpdir is reset for each easystack item""" + + orig_tmpdir_tempfile = tempfile.gettempdir() + orig_tmpdir_env = os.getenv('TMPDIR') + orig_tmpdir_tempfile_len = len(orig_tmpdir_env.split(os.path.sep)) + orig_tmpdir_env_len = len(orig_tmpdir_env.split(os.path.sep)) + test_es_txt = '\n'.join([ "easyconfigs:", " - toy-0.0-gompi-2018a.eb:", " - libtoy-0.0.eb:", + # also include a couple of easyconfigs for which a module is already available in test environment, + # see test/framework/modules + " - GCC-7.3.0-2.30", + " - FFTW-3.3.7-gompi-2018a", + " - foss-2018a", ]) test_es_path = os.path.join(self.test_prefix, 'test.yml') write_file(test_es_path, test_es_txt) @@ -143,10 +155,24 @@ def test_easystack_restore_env_after_each_build(self): '--easystack', test_es_path ] - stdout = self.eb_main(args, do_build=True, raise_error=True) + stdout = self.eb_main(args, do_build=True, raise_error=True, reset_env=False, redo_init_config=False) regex = re.compile(r"WARNING Loaded modules detected: \[.*gompi/2018.*\]\n") self.assertFalse(regex.search(stdout), "Pattern '%s' should not be found in: %s" % (regex.pattern, stdout)) + # temporary directory after run should be exactly 2 levels deeper than original one: + # - 1 level added by setting up configuration in EasyBuild main function + # - 1 extra level added by first re-configuration for easystack item + # (because $TMPDIR set by configuration done in main function is retained) + tmpdir_tempfile = tempfile.gettempdir() + tmpdir_env = os.getenv('TMPDIR') + tmpdir_tempfile_len = len(tmpdir_env.split(os.path.sep)) + tmpdir_env_len = len(tmpdir_env.split(os.path.sep)) + + self.assertEqual(tmpdir_tempfile_len, orig_tmpdir_tempfile_len + 2) + self.assertEqual(tmpdir_env_len, orig_tmpdir_env_len + 2) + self.assertTrue(tmpdir_tempfile.startswith(orig_tmpdir_tempfile)) + self.assertTrue(tmpdir_env.startswith(orig_tmpdir_env)) + def test_missing_easyconfigs_key(self): """Test that EasyStack file that doesn't contain an EasyConfigs key will fail with sane error message""" topdir = os.path.dirname(os.path.abspath(__file__)) From 7a1a56d0a7f94af13ed019c95115d7e046e51a9f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 27 Sep 2023 15:42:01 +0200 Subject: [PATCH 135/312] fix mistake in test_easystack_restore_env_after_each_build when determine length of tmpdir path in tempfile Co-authored-by: ocaisa --- test/framework/easystack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easystack.py b/test/framework/easystack.py index 6951857129..91efcc94b3 100644 --- a/test/framework/easystack.py +++ b/test/framework/easystack.py @@ -165,7 +165,7 @@ def test_easystack_restore_env_after_each_build(self): # (because $TMPDIR set by configuration done in main function is retained) tmpdir_tempfile = tempfile.gettempdir() tmpdir_env = os.getenv('TMPDIR') - tmpdir_tempfile_len = len(tmpdir_env.split(os.path.sep)) + tmpdir_tempfile_len = len(tmpdir_tempfile.split(os.path.sep)) tmpdir_env_len = len(tmpdir_env.split(os.path.sep)) self.assertEqual(tmpdir_tempfile_len, orig_tmpdir_tempfile_len + 2) From e4b2ac353364b02152cc614ee2089bf8eaf026a5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 27 Sep 2023 15:52:03 +0200 Subject: [PATCH 136/312] minor tweaks to CI workflow for running EasyBuild framework unit test suite --- .github/workflows/unit_tests.yml | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index d2f8ec0ae9..5b09fc654b 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -103,16 +103,11 @@ jobs: # and are only run after the PR gets merged GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}} run: | - # don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools, - # and only when testing with Lua as module syntax, - # to avoid hitting GitHub rate limit; + # don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools, to avoid hitting GitHub rate limit; # tests that require a GitHub token are skipped automatically when no GitHub token is available if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]]; then if [ ! -z $GITHUB_TOKEN ]; then - if [ "x${{matrix.python}}" == 'x2.6' ]; - then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())"; - else SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"; - fi; + SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"; python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; fi echo "GitHub token installed!" @@ -174,17 +169,18 @@ jobs: else export EASYBUILD_MODULES_TOOL=Lmod fi - export TEST_EASYBUILD_MODULES_TOOL=$EASYBUILD_MODULES_TOOL + export TEST_EASYBUILD_MODULES_TOOL=${EASYBUILD_MODULES_TOOL} # Run tests with LUA and Tcl module syntax (where supported) for module_syntax in Lua Tcl; do # Only Lmod supports Lua - if [[ "$module_syntax" == "Lua" ]] && [[ "$EASYBUILD_MODULES_TOOL" != "Lmod" ]]; then + if [[ "${module_syntax}" == "Lua" ]] && [[ "${EASYBUILD_MODULES_TOOL}" != "Lmod" ]]; then + echo "Not testing with '${module_syntax}' as module syntax with '${EASYBUILD_MODULES_TOOL}' as modules tool" continue fi printf '\n\n=====================> Using $module_syntax module syntax <=====================\n\n' - export EASYBUILD_MODULE_SYNTAX="$module_syntax" - export TEST_EASYBUILD_MODULE_SYNTAX="$EASYBUILD_MODULE_SYNTAX" + export EASYBUILD_MODULE_SYNTAX="${module_syntax}" + export TEST_EASYBUILD_MODULE_SYNTAX="${EASYBUILD_MODULE_SYNTAX}" eb --show-config # gather some useful info on test system @@ -198,7 +194,7 @@ jobs: python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log # try and make sure output of running tests is clean (no printed messages/warnings) IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test" - # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches) + # '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches) PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1) done From 46c686fd010e1f941097297b9ae2a0ee4644461b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Oct 2023 15:53:12 +0200 Subject: [PATCH 137/312] let run_shell_cmd produce error report when shell command failed --- easybuild/tools/run.py | 59 ++++++++++++++++++++++++++++++++++++++++-- test/framework/run.py | 44 ++++++++++++++++++++++++++++--- 2 files changed, 98 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 85b79a7868..4a08220484 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -37,6 +37,7 @@ """ import contextlib import functools +import inspect import os import re import signal @@ -49,7 +50,8 @@ import easybuild.tools.asyncprocess as asyncprocess from easybuild.base import fancylogger -from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, time_str_since +from easybuild.base.exceptions import LoggedException +from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_error, print_msg, time_str_since from easybuild.tools.config import ERROR, IGNORE, WARN, build_option from easybuild.tools.hooks import RUN_SHELL_CMD, load_hooks, run_hook from easybuild.tools.utilities import trace_msg @@ -78,6 +80,56 @@ RunResult = namedtuple('RunResult', ('cmd', 'exit_code', 'output', 'stderr', 'work_dir')) +def report_run_shell_cmd_error(cmd, exit_code, work_dir, output, stderr): + """ + Report error that occurred when running a shell command. + """ + cmd_name = cmd.split(' ')[0] + + tmpdir = tempfile.mkdtemp(prefix='shell-cmd-error-') + output_fp = os.path.join(tmpdir, f"{cmd_name}.out") + with open(output_fp, 'w') as fp: + fp.write(output or '') + stderr_fp = os.path.join(tmpdir, f"{cmd_name}.err") + with open(stderr_fp, 'w') as fp: + fp.write(stderr or '') + + # figure out where failing command was run + # need to go 3 levels down: + # 1) this function + # 2) run_shell_cmd function + # 3) run_cmd_cache decorator + # 4) actual caller site + frameinfo = inspect.getouterframes(inspect.currentframe())[3] + caller_file_name = frameinfo.filename + caller_line_nr = frameinfo.lineno + caller_function_name = frameinfo.function + + error_info = [ + f"| full shell command | {cmd}", + f"| exit code | {exit_code}", + f"| working directory | {work_dir}", + ] + if stderr is None: + error_info.append(f"| output (stdout + stderr) | {output_fp}") + else: + error_info.extend([ + f"| output (stdout) | {output_fp}", + f"| error/warnings (stderr) | {stderr_fp}", + ]) + + called_from_info = f"{caller_function_name} function in {caller_file_name} (line {caller_line_nr})" + error_info.append(f"| called from | {called_from_info}") + + error_msg = '\n'.join([''] + error_info + [ + '', + f"ERROR: shell command '{cmd_name}' failed!", + '', + ]) + sys.stderr.write(error_msg) + sys.exit(exit_code) + + def run_cmd_cache(func): """Function decorator to cache (and retrieve cached) results of running commands.""" cache = {} @@ -204,7 +256,7 @@ def to_cmd_str(cmd): _log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd) _log.info(f"Running command '{cmd_str}' in {work_dir}") - proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=stderr, check=fail_on_error, + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=stderr, check=False, cwd=work_dir, env=env, input=stdin, shell=shell, executable=executable) # return output as a regular string rather than a byte sequence (and non-UTF-8 characters get stripped out) @@ -213,6 +265,9 @@ def to_cmd_str(cmd): res = RunResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr_output, work_dir=work_dir) + if res.exit_code != 0 and fail_on_error: + report_run_shell_cmd_error(res.cmd, res.exit_code, res.work_dir, output=res.output, stderr=res.stderr) + if with_hooks: run_hook_kwargs = { 'exit_code': res.exit_code, diff --git a/test/framework/run.py b/test/framework/run.py index 4d6974bb04..0ae3b23396 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -49,7 +49,7 @@ import easybuild.tools.utilities from easybuild.tools.build_log import EasyBuildError, init_logging, stop_logging from easybuild.tools.config import update_build_option -from easybuild.tools.filetools import adjust_permissions, mkdir, read_file, write_file +from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir, read_file, write_file from easybuild.tools.run import check_async_cmd, check_log_for_errors, complete_cmd, get_output_from_process from easybuild.tools.run import RunResult, parse_log_for_error, run_shell_cmd, run_cmd, run_cmd_qa, subprocess_terminate from easybuild.tools.config import ERROR, IGNORE, WARN @@ -317,9 +317,47 @@ def handler(signum, _): signal.signal(signal.SIGALRM, handler) signal.alarm(3) - with self.mocked_stdout_stderr(): - res = run_shell_cmd("kill -9 $$", fail_on_error=False) + # command to kill parent shell + cmd = "kill -9 $$" + + workdir = os.path.realpath(self.test_prefix) + change_dir(workdir) + + with self.mocked_stdout_stderr() as (_, stderr): + self.assertErrorRegex(SystemExit, '.*', run_shell_cmd, cmd) + + # check error reporting output + stderr = stderr.getvalue() + patterns = [ + r"^\| full shell command[ ]*\| kill -9 \$\$", + r"^\| exit code[ ]*\| -9", + r"^\| working directory[ ]*\| " + workdir, + r"^\| called from[ ]*\| assertErrorRegex function in .*/easybuild/base/testing.py \(line [0-9]+\)", + r"^ERROR: shell command 'kill' failed!", + r"^\| output \(stdout \+ stderr\)[ ]*\| .*/shell-cmd-error-.*/kill.out", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stderr), "Pattern '%s' should be found in: %s" % (regex.pattern, stderr)) + + # check error reporting output when stdout/stderr are collected separately + with self.mocked_stdout_stderr() as (_, stderr): + self.assertErrorRegex(SystemExit, '.*', run_shell_cmd, cmd, split_stderr=True) + stderr = stderr.getvalue() + patterns.pop(-1) + patterns.extend([ + r"^\| output \(stdout\)[ ]*\| .*/shell-cmd-error-.*/kill.out", + r"^\| error/warnings \(stderr\)[ ]*\| .*/shell-cmd-error-.*/kill.err", + ]) + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stderr), "Pattern '%s' should be found in: %s" % (regex.pattern, stderr)) + + # no error reporting when fail_on_error is disabled + with self.mocked_stdout_stderr() as (_, stderr): + res = run_shell_cmd(cmd, fail_on_error=False) self.assertEqual(res.exit_code, -9) + self.assertEqual(stderr.getvalue(), '') finally: # cleanup: disable the alarm + reset signal handler for SIGALRM From 07c85f289eb9151e901707ff9164f1486a373de3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Oct 2023 16:43:44 +0200 Subject: [PATCH 138/312] rename RunResult to RunShellCmdResult --- easybuild/tools/run.py | 8 ++++---- test/framework/run.py | 10 ++++++---- test/framework/systemtools.py | 7 ++++--- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 4a08220484..b48e6a8a89 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -77,7 +77,7 @@ ] -RunResult = namedtuple('RunResult', ('cmd', 'exit_code', 'output', 'stderr', 'work_dir')) +RunShellCmdResult = namedtuple('RunShellCmdResult', ('cmd', 'exit_code', 'output', 'stderr', 'work_dir')) def report_run_shell_cmd_error(cmd, exit_code, work_dir, output, stderr): @@ -230,7 +230,7 @@ def to_cmd_str(cmd): msg += f" (in {work_dir})" dry_run_msg(msg, silent=silent) - return RunResult(cmd=cmd_str, exit_code=0, output='', stderr=None, work_dir=work_dir) + return RunShellCmdResult(cmd=cmd_str, exit_code=0, output='', stderr=None, work_dir=work_dir) start_time = datetime.now() if not hidden: @@ -261,9 +261,9 @@ def to_cmd_str(cmd): # return output as a regular string rather than a byte sequence (and non-UTF-8 characters get stripped out) output = proc.stdout.decode('utf-8', 'ignore') - stderr_output = proc.stderr.decode('utf-8', 'ignore') if split_stderr else None + stderr= proc.stderr.decode('utf-8', 'ignore') if split_stderr else None - res = RunResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr_output, work_dir=work_dir) + res = RunShellCmdResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr, work_dir=work_dir) if res.exit_code != 0 and fail_on_error: report_run_shell_cmd_error(res.cmd, res.exit_code, res.work_dir, output=res.output, stderr=res.stderr) diff --git a/test/framework/run.py b/test/framework/run.py index 0ae3b23396..609f74ebcc 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -50,8 +50,9 @@ from easybuild.tools.build_log import EasyBuildError, init_logging, stop_logging from easybuild.tools.config import update_build_option from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir, read_file, write_file -from easybuild.tools.run import check_async_cmd, check_log_for_errors, complete_cmd, get_output_from_process -from easybuild.tools.run import RunResult, parse_log_for_error, run_shell_cmd, run_cmd, run_cmd_qa, subprocess_terminate +from easybuild.tools.run import RunShellCmdResult, check_async_cmd, check_log_for_errors, complete_cmd +from easybuild.tools.run import get_output_from_process, parse_log_for_error, run_shell_cmd, run_cmd, run_cmd_qa +from easybuild.tools.run import subprocess_terminate from easybuild.tools.config import ERROR, IGNORE, WARN @@ -863,7 +864,8 @@ def test_run_shell_cmd_cache(self): # inject value into cache to check whether executing command again really returns cached value with self.mocked_stdout_stderr(): - cached_res = RunResult(cmd=cmd, output="123456", exit_code=123, stderr=None, work_dir='/test_ulimit') + cached_res = RunShellCmdResult(cmd=cmd, output="123456", exit_code=123, stderr=None, + work_dir='/test_ulimit') run_shell_cmd.update_cache({(cmd, None): cached_res}) res = run_shell_cmd(cmd) self.assertEqual(res.cmd, cmd) @@ -881,7 +883,7 @@ def test_run_shell_cmd_cache(self): # inject different output for cat with 'foo' as stdin to check whether cached value is used with self.mocked_stdout_stderr(): - cached_res = RunResult(cmd=cmd, output="bar", exit_code=123, stderr=None, work_dir='/test_cat') + cached_res = RunShellCmdResult(cmd=cmd, output="bar", exit_code=123, stderr=None, work_dir='/test_cat') run_shell_cmd.update_cache({(cmd, 'foo'): cached_res}) res = run_shell_cmd(cmd, stdin='foo') self.assertEqual(res.cmd, cmd) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index c86427326a..63bcfffe2e 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -40,7 +40,7 @@ import easybuild.tools.systemtools as st from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import adjust_permissions, read_file, symlink, which, write_file -from easybuild.tools.run import RunResult, run_shell_cmd +from easybuild.tools.run import RunShellCmdResult, run_shell_cmd from easybuild.tools.systemtools import CPU_ARCHITECTURES, AARCH32, AARCH64, POWER, X86_64 from easybuild.tools.systemtools import CPU_FAMILIES, POWER_LE, DARWIN, LINUX, UNKNOWN from easybuild.tools.systemtools import CPU_VENDORS, AMD, APM, ARM, CAVIUM, IBM, INTEL @@ -340,7 +340,7 @@ def mocked_run_shell_cmd(cmd, **kwargs): "ulimit -u": '40', } if cmd in known_cmds: - return RunResult(cmd=cmd, exit_code=0, output=known_cmds[cmd], stderr=None, work_dir=os.getcwd()) + return RunShellCmdResult(cmd=cmd, exit_code=0, output=known_cmds[cmd], stderr=None, work_dir=os.getcwd()) else: return run_shell_cmd(cmd, **kwargs) @@ -774,7 +774,8 @@ def test_gcc_version_darwin(self): """Test getting gcc version (mocked for Darwin).""" st.get_os_type = lambda: st.DARWIN out = "Apple LLVM version 7.0.0 (clang-700.1.76)" - mocked_run_res = RunResult(cmd="gcc --version", exit_code=0, output=out, stderr=None, work_dir=os.getcwd()) + cwd = os.getcwd() + mocked_run_res = RunShellCmdResult(cmd="gcc --version", exit_code=0, output=out, stderr=None, work_dir=cwd) st.run_shell_cmd = lambda *args, **kwargs: mocked_run_res self.assertEqual(get_gcc_version(), None) From 2119ff4bc393f32388494ec5f09f168766dc09be Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Oct 2023 18:04:15 +0200 Subject: [PATCH 139/312] separate EasyBuild crashes (unexpected exception being raised) from EasyBuildError being raised, add crash hook, improve general error reporting --- easybuild/main.py | 19 ++++++++++++------- easybuild/tools/build_log.py | 2 +- easybuild/tools/hooks.py | 2 ++ test/framework/build_log.py | 8 ++++---- test/framework/options.py | 1 + 5 files changed, 20 insertions(+), 12 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index d6d5755e9a..ff94cf0aec 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -70,13 +70,14 @@ from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, merge_pr, new_branch_github, new_pr from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr -from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, START, END, CANCEL, FAIL +from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, START, END, CANCEL, CRASH, FAIL from easybuild.tools.hooks import load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import opts_dict_to_eb_opts, set_up_configuration, use_color from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs +from easybuild.tools.run import RunShellCmdError from easybuild.tools.package.utilities import check_pkg_support from easybuild.tools.parallelbuild import submit_jobs from easybuild.tools.repository.repository import init_repository @@ -149,11 +150,11 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): # keep track of success/total count if ec_res['success']: - test_msg = "Successfully built %s" % ec['spec'] + test_msg = "Successfully installed %s" % ec['spec'] else: - test_msg = "Build of %s failed" % ec['spec'] + test_msg = "Installation of %s failed" % os.path.basename(ec['spec']) if 'err' in ec_res: - test_msg += " (err: %s)" % ec_res['err'] + test_msg += ": %s" % ec_res['err'] # dump test report next to log file test_report_txt = create_test_report(test_msg, [(ec, ec_res)], init_session_state) @@ -169,8 +170,8 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): adjust_permissions(parent_dir, stat.S_IWUSR, add=False, recursive=False) if not ec_res['success'] and exit_on_failure: - if 'traceback' in ec_res: - raise EasyBuildError(ec_res['traceback']) + if not isinstance(ec_res['err'], EasyBuildError): + raise ec_res['err'] else: raise EasyBuildError(test_msg) @@ -754,7 +755,11 @@ def prepare_main(args=None, logfile=None, testing=None): main(prepared_cfg_data=(init_session_state, eb_go, cfg_settings)) except EasyBuildError as err: run_hook(FAIL, hooks, args=[err]) - print_error(err.msg) + sys.exit(1) except KeyboardInterrupt as err: run_hook(CANCEL, hooks, args=[err]) print_error("Cancelled by user: %s" % err) + except Exception as err: + run_hook(CRASH, hooks, args=[err]) + sys.stderr.write("EasyBuild crashed! Please consider reporting a bug, this should not happen...\n\n") + raise diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index 215b8d38aa..e8a479e24e 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -167,7 +167,7 @@ def nosupport(self, msg, ver): def error(self, msg, *args, **kwargs): """Print error message and raise an EasyBuildError.""" - ebmsg = "EasyBuild crashed with an error %s: " % self.caller_info() + ebmsg = "EasyBuild encountered an error %s: " % self.caller_info() fancylogger.FancyLogger.error(self, ebmsg + msg, *args, **kwargs) def devel(self, msg, *args, **kwargs): diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 319b864398..2bb329027b 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -67,6 +67,7 @@ END = 'end' CANCEL = 'cancel' +CRASH = 'crash' FAIL = 'fail' RUN_SHELL_CMD = 'run_shell_cmd' @@ -107,6 +108,7 @@ POST_PREF + BUILD_AND_INSTALL_LOOP, END, CANCEL, + CRASH, FAIL, PRE_PREF + RUN_SHELL_CMD, POST_PREF + RUN_SHELL_CMD, diff --git a/test/framework/build_log.py b/test/framework/build_log.py index a1792b998f..21755b62b3 100644 --- a/test/framework/build_log.py +++ b/test/framework/build_log.py @@ -139,8 +139,8 @@ def test_easybuildlog(self): r"fancyroot.test_easybuildlog \[WARNING\] :: Deprecated functionality.*onemorewarning.*", r"fancyroot.test_easybuildlog \[WARNING\] :: Deprecated functionality.*lastwarning.*", r"fancyroot.test_easybuildlog \[WARNING\] :: Deprecated functionality.*thisisnotprinted.*", - r"fancyroot.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput", - r"fancyroot.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): err: msg: %s", + r"fancyroot.test_easybuildlog \[ERROR\] :: EasyBuild encountered an error \(at .* in .*\): kaput", + r"fancyroot.test_easybuildlog \[ERROR\] :: EasyBuild encountered an error \(at .* in .*\): err: msg: %s", r"fancyroot.test_easybuildlog \[ERROR\] :: .*EasyBuild encountered an exception \(at .* in .*\): oops", '', ]) @@ -168,7 +168,7 @@ def test_easybuildlog(self): r"fancyroot.test_easybuildlog \[WARNING\] :: bleh", r"fancyroot.test_easybuildlog \[INFO\] :: 4\+2 = 42", r"fancyroot.test_easybuildlog \[DEBUG\] :: this is just a test", - r"fancyroot.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): foo baz baz", + r"fancyroot.test_easybuildlog \[ERROR\] :: EasyBuild encountered an error \(at .* in .*\): foo baz baz", '', ]) logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M) @@ -223,7 +223,7 @@ def test_log_levels(self): info_msg = r"%s \[INFO\] :: fyi" % prefix warning_msg = r"%s \[WARNING\] :: this is a warning" % prefix deprecated_msg = r"%s \[WARNING\] :: Deprecated functionality, .*: almost kaput; see .*" % prefix - error_msg = r"%s \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput" % prefix + error_msg = r"%s \[ERROR\] :: EasyBuild encountered an error \(at .* in .*\): kaput" % prefix expected_logtxt = '\n'.join([ error_msg, diff --git a/test/framework/options.py b/test/framework/options.py index bde1e565c5..f6742131bf 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -737,6 +737,7 @@ def test_avail_hooks(self): " post_build_and_install_loop_hook", " end_hook", " cancel_hook", + " crash_hook", " fail_hook", " pre_run_shell_cmd_hook", " post_run_shell_cmd_hook", From df3b72c53eb55e7b8b977bd8cac14ddc17cb3596 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Oct 2023 18:34:18 +0200 Subject: [PATCH 140/312] separate raising of RunShellCmdError from printing info on failed shell command, log command output before log message that mentions success of failure for command --- easybuild/tools/run.py | 112 +++++++++++++++++++++++++++-------------- test/framework/run.py | 107 +++++++++++++++++++++++++++------------ 2 files changed, 148 insertions(+), 71 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index b48e6a8a89..fb31e9db35 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -80,19 +80,69 @@ RunShellCmdResult = namedtuple('RunShellCmdResult', ('cmd', 'exit_code', 'output', 'stderr', 'work_dir')) -def report_run_shell_cmd_error(cmd, exit_code, work_dir, output, stderr): +class RunShellCmdError(BaseException): + + def __init__(self, cmd, exit_code, work_dir, output, stderr, caller_info, *args, **kwargs): + """Constructor for RunShellCmdError.""" + self.cmd = cmd + self.cmd_name = cmd.split(' ')[0] + self.exit_code = exit_code + self.work_dir = work_dir + self.output = output + self.stderr = stderr + self.caller_info = caller_info + + msg = f"Shell command '{self.cmd_name}' failed!" + super(RunShellCmdError, self).__init__(msg, *args, **kwargs) + + +def print_run_shell_cmd_error(err): """ - Report error that occurred when running a shell command. + Report failed shell command using provided RunShellCmdError instance """ - cmd_name = cmd.split(' ')[0] + + def pad_4_spaces(msg): + return ' ' * 4 + msg + + cmd_name = err.cmd.split(' ')[0] + error_info = [ + '', + f"ERROR: Shell command failed!", + pad_4_spaces(f"full command -> {err.cmd}"), + pad_4_spaces(f"exit code -> {err.exit_code}"), + pad_4_spaces(f"working directory -> {err.work_dir}"), + ] tmpdir = tempfile.mkdtemp(prefix='shell-cmd-error-') output_fp = os.path.join(tmpdir, f"{cmd_name}.out") with open(output_fp, 'w') as fp: - fp.write(output or '') - stderr_fp = os.path.join(tmpdir, f"{cmd_name}.err") - with open(stderr_fp, 'w') as fp: - fp.write(stderr or '') + fp.write(err.output or '') + + if err.stderr is None: + error_info.append(pad_4_spaces(f"output (stdout + stderr) -> {output_fp}")) + else: + stderr_fp = os.path.join(tmpdir, f"{cmd_name}.err") + with open(stderr_fp, 'w') as fp: + fp.write(err.stderr) + error_info.extend([ + pad_4_spaces(f"output (stdout) -> {output_fp}"), + pad_4_spaces(f"error/warnings (stderr) -> {stderr_fp}"), + ]) + + caller_file_name, caller_line_nr, caller_function_name = err.caller_info + called_from_info = f"'{caller_function_name}' function in {caller_file_name} (line {caller_line_nr})" + error_info.extend([ + pad_4_spaces(f"called from -> {called_from_info}"), + '', + ]) + + sys.stderr.write('\n'.join(error_info) + '\n') + + +def raise_run_shell_cmd_error(cmd, exit_code, work_dir, output, stderr): + """ + Raise RunShellCmdError for failing shell command, after collecting additional caller info + """ # figure out where failing command was run # need to go 3 levels down: @@ -104,30 +154,9 @@ def report_run_shell_cmd_error(cmd, exit_code, work_dir, output, stderr): caller_file_name = frameinfo.filename caller_line_nr = frameinfo.lineno caller_function_name = frameinfo.function + caller_info = (frameinfo.filename, frameinfo.lineno, frameinfo.function) - error_info = [ - f"| full shell command | {cmd}", - f"| exit code | {exit_code}", - f"| working directory | {work_dir}", - ] - if stderr is None: - error_info.append(f"| output (stdout + stderr) | {output_fp}") - else: - error_info.extend([ - f"| output (stdout) | {output_fp}", - f"| error/warnings (stderr) | {stderr_fp}", - ]) - - called_from_info = f"{caller_function_name} function in {caller_file_name} (line {caller_line_nr})" - error_info.append(f"| called from | {called_from_info}") - - error_msg = '\n'.join([''] + error_info + [ - '', - f"ERROR: shell command '{cmd_name}' failed!", - '', - ]) - sys.stderr.write(error_msg) - sys.exit(exit_code) + raise RunShellCmdError(cmd, exit_code, work_dir, output, stderr, caller_info) def run_cmd_cache(func): @@ -265,8 +294,20 @@ def to_cmd_str(cmd): res = RunShellCmdResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr, work_dir=work_dir) - if res.exit_code != 0 and fail_on_error: - report_run_shell_cmd_error(res.cmd, res.exit_code, res.work_dir, output=res.output, stderr=res.stderr) + # always log command output + cmd_name = cmd_str.split(' ')[0] + if split_stderr: + _log.info(f"Output of '{cmd_name} ...' shell command (stdout only):\n{res.output}") + _log.info(f"Warnings and errors of '{cmd_name} ...' shell command (stderr only):\n{res.stderr}") + else: + _log.info(f"Output of '{cmd_name} ...' shell command (stdout + stderr):\n{res.output}") + + if res.exit_code == 0: + _log.info(f"Shell command completed successfully (see output above): {cmd_str}") + else: + _log.warning(f"Shell command FAILED (exit code {res.exit_code}, see output above): {cmd_str}") + if fail_on_error: + raise_run_shell_cmd_error(res.cmd, res.exit_code, res.work_dir, output=res.output, stderr=res.stderr) if with_hooks: run_hook_kwargs = { @@ -277,13 +318,6 @@ def to_cmd_str(cmd): } run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs) - if split_stderr: - log_msg = f"Command '{cmd_str}' exited with exit code {res.exit_code}, " - log_msg += f"with stdout:\n{res.output}\nstderr:\n{res.stderr}" - else: - log_msg = f"Command '{cmd_str}' exited with exit code {res.exit_code} and output:\n{res.output}" - _log.info(log_msg) - if not hidden: time_since_start = time_str_since(start_time) trace_msg(f"command completed: exit {res.exit_code}, ran in {time_since_start}") diff --git a/test/framework/run.py b/test/framework/run.py index 609f74ebcc..0e4de57cf7 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -50,9 +50,9 @@ from easybuild.tools.build_log import EasyBuildError, init_logging, stop_logging from easybuild.tools.config import update_build_option from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir, read_file, write_file -from easybuild.tools.run import RunShellCmdResult, check_async_cmd, check_log_for_errors, complete_cmd -from easybuild.tools.run import get_output_from_process, parse_log_for_error, run_shell_cmd, run_cmd, run_cmd_qa -from easybuild.tools.run import subprocess_terminate +from easybuild.tools.run import RunShellCmdResult, RunShellCmdError, check_async_cmd, check_log_for_errors +from easybuild.tools.run import complete_cmd, get_output_from_process, parse_log_for_error +from easybuild.tools.run import print_run_shell_cmd_error, run_cmd, run_cmd_qa, run_shell_cmd, subprocess_terminate from easybuild.tools.config import ERROR, IGNORE, WARN @@ -311,6 +311,9 @@ def test_run_shell_cmd_fail(self): def handler(signum, _): raise RuntimeError("Signal handler called with signal %s" % signum) + # disable trace output for this test (so stdout remains empty) + update_build_option('trace', False) + orig_sigalrm_handler = signal.getsignal(signal.SIGALRM) try: @@ -321,38 +324,78 @@ def handler(signum, _): # command to kill parent shell cmd = "kill -9 $$" - workdir = os.path.realpath(self.test_prefix) - change_dir(workdir) + work_dir = os.path.realpath(self.test_prefix) + change_dir(work_dir) - with self.mocked_stdout_stderr() as (_, stderr): - self.assertErrorRegex(SystemExit, '.*', run_shell_cmd, cmd) - - # check error reporting output - stderr = stderr.getvalue() - patterns = [ - r"^\| full shell command[ ]*\| kill -9 \$\$", - r"^\| exit code[ ]*\| -9", - r"^\| working directory[ ]*\| " + workdir, - r"^\| called from[ ]*\| assertErrorRegex function in .*/easybuild/base/testing.py \(line [0-9]+\)", - r"^ERROR: shell command 'kill' failed!", - r"^\| output \(stdout \+ stderr\)[ ]*\| .*/shell-cmd-error-.*/kill.out", - ] - for pattern in patterns: - regex = re.compile(pattern, re.M) - self.assertTrue(regex.search(stderr), "Pattern '%s' should be found in: %s" % (regex.pattern, stderr)) + try: + run_shell_cmd(cmd) + self.assertFalse("This should never be reached, RunShellCmdError should occur!") + except RunShellCmdError as err: + self.assertEqual(str(err), "Shell command 'kill' failed!") + self.assertEqual(err.cmd, "kill -9 $$") + self.assertEqual(err.cmd_name, 'kill') + self.assertEqual(err.exit_code, -9) + self.assertEqual(err.work_dir, work_dir) + self.assertEqual(err.output, '') + self.assertEqual(err.stderr, None) + self.assertTrue(isinstance(err.caller_info, tuple)) + self.assertEqual(len(err.caller_info), 3) + self.assertEqual(err.caller_info[0], __file__) + self.assertTrue(isinstance(err.caller_info[1], int)) # line number of calling site + self.assertEqual(err.caller_info[2], 'test_run_shell_cmd_fail') + + with self.mocked_stdout_stderr() as (_, stderr): + print_run_shell_cmd_error(err) + + # check error reporting output + stderr = stderr.getvalue() + patterns = [ + r"^ERROR: Shell command failed!", + r"^\s+full command\s* -> kill -9 \$\$", + r"^\s+exit code\s* -> -9", + r"^\s+working directory\s* -> " + work_dir, + r"^\s+called from\s* -> 'test_run_shell_cmd_fail' function in .*/test/.*/run.py \(line [0-9]+\)", + r"^\s+output \(stdout \+ stderr\)\s* -> .*/shell-cmd-error-.*/kill.out", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stderr), "Pattern '%s' should be found in: %s" % (pattern, stderr)) # check error reporting output when stdout/stderr are collected separately - with self.mocked_stdout_stderr() as (_, stderr): - self.assertErrorRegex(SystemExit, '.*', run_shell_cmd, cmd, split_stderr=True) - stderr = stderr.getvalue() - patterns.pop(-1) - patterns.extend([ - r"^\| output \(stdout\)[ ]*\| .*/shell-cmd-error-.*/kill.out", - r"^\| error/warnings \(stderr\)[ ]*\| .*/shell-cmd-error-.*/kill.err", - ]) - for pattern in patterns: - regex = re.compile(pattern, re.M) - self.assertTrue(regex.search(stderr), "Pattern '%s' should be found in: %s" % (regex.pattern, stderr)) + try: + run_shell_cmd(cmd, split_stderr=True) + self.assertFalse("This should never be reached, RunShellCmdError should occur!") + except RunShellCmdError as err: + self.assertEqual(str(err), "Shell command 'kill' failed!") + self.assertEqual(err.cmd, "kill -9 $$") + self.assertEqual(err.cmd_name, 'kill') + self.assertEqual(err.exit_code, -9) + self.assertEqual(err.work_dir, work_dir) + self.assertEqual(err.output, '') + self.assertEqual(err.stderr, '') + self.assertTrue(isinstance(err.caller_info, tuple)) + self.assertEqual(len(err.caller_info), 3) + self.assertEqual(err.caller_info[0], __file__) + self.assertTrue(isinstance(err.caller_info[1], int)) # line number of calling site + self.assertEqual(err.caller_info[2], 'test_run_shell_cmd_fail') + + with self.mocked_stdout_stderr() as (_, stderr): + print_run_shell_cmd_error(err) + + # check error reporting output + stderr = stderr.getvalue() + patterns = [ + r"^ERROR: Shell command failed!", + r"^\s+full command\s+ -> kill -9 \$\$", + r"^\s+exit code\s+ -> -9", + r"^\s+working directory\s+ -> " + work_dir, + r"^\s+called from\s+ -> 'test_run_shell_cmd_fail' function in .*/test/.*/run.py \(line [0-9]+\)", + r"^\s+output \(stdout\)\s+ -> .*/shell-cmd-error-.*/kill.out", + r"^\s+error/warnings \(stderr\)\s+ -> .*/shell-cmd-error-.*/kill.err", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stderr), "Pattern '%s' should be found in: %s" % (pattern, stderr)) # no error reporting when fail_on_error is disabled with self.mocked_stdout_stderr() as (_, stderr): From 0b7f3514e1e12cf30285a32b2ab2a150934082f7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Oct 2023 18:35:13 +0200 Subject: [PATCH 141/312] catch RunShellCmdError in EasyBlock.run_all_steps to do more meaningful error reporting incl. step name and easyconfig filename --- easybuild/framework/easyblock.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index e86d475f64..79d32ff20a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -87,7 +87,7 @@ from easybuild.tools.hooks import MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP from easybuild.tools.hooks import POSTPROC_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP from easybuild.tools.hooks import SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook -from easybuild.tools.run import check_async_cmd, run_cmd +from easybuild.tools.run import RunShellCmdError, check_async_cmd, print_run_shell_cmd_error, run_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version @@ -4124,6 +4124,11 @@ def run_all_steps(self, run_test_cases): start_time = datetime.now() try: self.run_step(step_name, step_methods) + except RunShellCmdError as err: + print_run_shell_cmd_error(err) + ec_path = os.path.basename(self.cfg.path) + error_msg = f"shell command '{err.cmd_name} ...' failed in {step_name} step for {ec_path}" + raise EasyBuildError(error_msg) finally: if not self.dry_run: step_duration = datetime.now() - start_time @@ -4225,7 +4230,7 @@ def build_and_install_one(ecdict, init_env): app.cfg['skip'] = skip # build easyconfig - errormsg = '(no error)' + error_msg = '(no error)' # timing info start_time = time.time() try: @@ -4263,9 +4268,7 @@ def build_and_install_one(ecdict, init_env): adjust_permissions(app.installdir, stat.S_IWUSR, add=False, recursive=True) except EasyBuildError as err: - first_n = 300 - errormsg = "build failed (first %d chars): %s" % (first_n, err.msg[:first_n]) - _log.warning(errormsg) + error_msg = err.msg result = False ended = 'ended' @@ -4387,11 +4390,7 @@ def ensure_writable_log_dir(log_dir): # build failed success = False summary = 'FAILED' - - build_dir = '' - if app.builddir: - build_dir = " (build directory: %s)" % (app.builddir) - succ = "unsuccessfully%s: %s" % (build_dir, errormsg) + succ = "unsuccessfully: " + error_msg # cleanup logs app.close_log() @@ -4424,7 +4423,7 @@ def ensure_writable_log_dir(log_dir): del app - return (success, application_log, errormsg) + return (success, application_log, error_msg) def copy_easyblocks_for_reprod(easyblock_instances, reprod_dir): From ce34aae3e45bd0dafeaf7c8bcf970d7683698d17 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Oct 2023 19:41:04 +0200 Subject: [PATCH 142/312] make hooks tests aware of crash hook --- test/framework/hooks.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/test/framework/hooks.py b/test/framework/hooks.py index 152d2352a4..c8e5d34583 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -79,6 +79,9 @@ def setUp(self): '', 'def fail_hook(err):', ' print("EasyBuild FAIL: %s" % err)', + '', + 'def crash_hook(err):', + ' print("EasyBuild CRASHED, oh no! => %s" % err)', ]) write_file(self.test_hooks_pymod, test_hooks_pymod_txt) @@ -97,8 +100,9 @@ def test_load_hooks(self): hooks = load_hooks(self.test_hooks_pymod) - self.assertEqual(len(hooks), 8) + self.assertEqual(len(hooks), 9) expected = [ + 'crash_hook', 'fail_hook', 'parse_hook', 'post_configure_hook', @@ -140,6 +144,7 @@ def test_find_hook(self): pre_single_extension_hook = [hooks[k] for k in hooks if k == 'pre_single_extension_hook'][0] start_hook = [hooks[k] for k in hooks if k == 'start_hook'][0] pre_run_shell_cmd_hook = [hooks[k] for k in hooks if k == 'pre_run_shell_cmd_hook'][0] + crash_hook = [hooks[k] for k in hooks if k == 'crash_hook'][0] fail_hook = [hooks[k] for k in hooks if k == 'fail_hook'][0] pre_build_and_install_loop_hook = [hooks[k] for k in hooks if k == 'pre_build_and_install_loop_hook'][0] @@ -175,6 +180,10 @@ def test_find_hook(self): self.assertEqual(find_hook('fail', hooks, pre_step_hook=True), None) self.assertEqual(find_hook('fail', hooks, post_step_hook=True), None) + self.assertEqual(find_hook('crash', hooks), crash_hook) + self.assertEqual(find_hook('crash', hooks, pre_step_hook=True), None) + self.assertEqual(find_hook('crash', hooks, post_step_hook=True), None) + hook_name = 'build_and_install_loop' self.assertEqual(find_hook(hook_name, hooks), None) self.assertEqual(find_hook(hook_name, hooks, pre_step_hook=True), pre_build_and_install_loop_hook) @@ -209,6 +218,7 @@ def run_hooks(): run_hook('single_extension', hooks, post_step_hook=True, args=[None]) run_hook('extensions', hooks, post_step_hook=True, args=[None]) run_hook('fail', hooks, args=[EasyBuildError('oops')]) + run_hook('crash', hooks, args=[RuntimeError('boom!')]) stdout = self.get_stdout() stderr = self.get_stderr() self.mock_stdout(False) @@ -244,6 +254,8 @@ def run_hooks(): "this is run before installing an extension", "== Running fail hook...", "EasyBuild FAIL: 'oops'", + "== Running crash hook...", + "EasyBuild CRASHED, oh no! => boom!", ] expected_stdout = '\n'.join(expected_stdout_lines) From 9d716d9bb7e44f3353d16cb70d1949cb9b795ee6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Oct 2023 19:41:37 +0200 Subject: [PATCH 143/312] add test that cover EasyBuild crashing --- easybuild/main.py | 10 +++++++--- test/framework/toy_build.py | 40 +++++++++++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index ff94cf0aec..98f76b9cd1 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -748,11 +748,11 @@ def prepare_main(args=None, logfile=None, testing=None): return init_session_state, eb_go, cfg_settings -if __name__ == "__main__": - init_session_state, eb_go, cfg_settings = prepare_main() +def main_with_hooks(args=None): + init_session_state, eb_go, cfg_settings = prepare_main(args=args) hooks = load_hooks(eb_go.options.hooks) try: - main(prepared_cfg_data=(init_session_state, eb_go, cfg_settings)) + main(args=args, prepared_cfg_data=(init_session_state, eb_go, cfg_settings)) except EasyBuildError as err: run_hook(FAIL, hooks, args=[err]) sys.exit(1) @@ -763,3 +763,7 @@ def prepare_main(args=None, logfile=None, testing=None): run_hook(CRASH, hooks, args=[err]) sys.stderr.write("EasyBuild crashed! Please consider reporting a bug, this should not happen...\n\n") raise + + +if __name__ == "__main__": + main_with_hooks() diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 2c4bef191c..5b83b98a62 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -42,7 +42,7 @@ import textwrap from easybuild.tools import LooseVersion from importlib import reload -from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, cleanup from test.framework.package import mock_fpm from unittest import TextTestRunner @@ -50,6 +50,7 @@ import easybuild.tools.module_naming_scheme # required to dynamically load test module naming scheme(s) from easybuild.framework.easyconfig.easyconfig import EasyConfig from easybuild.framework.easyconfig.parser import EasyConfigParser +from easybuild.main import main_with_hooks from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import get_module_syntax, get_repositorypath from easybuild.tools.environment import modify_env @@ -57,7 +58,7 @@ from easybuild.tools.filetools import read_file, remove_dir, remove_file, which, write_file from easybuild.tools.module_generator import ModuleGeneratorTcl from easybuild.tools.modules import Lmod -from easybuild.tools.run import run_cmd +from easybuild.tools.run import RunShellCmdError, run_cmd, run_shell_cmd from easybuild.tools.utilities import nub from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.version import VERSION as EASYBUILD_VERSION @@ -4068,6 +4069,41 @@ def test_toy_build_info_msg(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def test_eb_crash(self): + """ + Test behaviour when EasyBuild crashes, for example due to a buggy hook + """ + hooks_file = os.path.join(self.test_prefix, 'my_hooks.py') + hooks_file_txt = textwrap.dedent(""" + def pre_configure_hook(self, *args, **kwargs): + no_such_thing + """) + write_file(hooks_file, hooks_file_txt) + + topdir = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + toy_eb = os.path.join(topdir, 'test', 'framework', 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') + toy_ec = os.path.join(topdir, 'test', 'framework', 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + + args = [ + toy_ec, + f'--hooks={hooks_file}', + f'--force', + f'--installpath={self.test_prefix}', + f'--include-easyblocks={toy_eb}', + ] + + with self.mocked_stdout_stderr() as (_, stderr): + cleanup() + try: + main_with_hooks(args=args) + self.assertFalse("This should never be reached, main function should have crashed!") + except NameError as err: + self.assertEqual(str(err), "name 'no_such_thing' is not defined") + + regex = re.compile(r"EasyBuild crashed! Please consider reporting a bug, this should not happen") + stderr = stderr.getvalue() + self.assertTrue(regex.search(stderr), f"Pattern '{regex.pattern}' should be found in {stderr}") + def suite(): """ return all the tests in this file """ From 7d2d5379cdfbbd17e7d5fc7c69299920df23ac05 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Oct 2023 19:45:00 +0200 Subject: [PATCH 144/312] fix unused imports + trivial code style issues in main.py and tools/run.py --- easybuild/main.py | 1 - easybuild/tools/run.py | 8 ++------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 98f76b9cd1..0f2c69733c 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -77,7 +77,6 @@ from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs -from easybuild.tools.run import RunShellCmdError from easybuild.tools.package.utilities import check_pkg_support from easybuild.tools.parallelbuild import submit_jobs from easybuild.tools.repository.repository import init_repository diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index fb31e9db35..9ab09ade64 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -50,8 +50,7 @@ import easybuild.tools.asyncprocess as asyncprocess from easybuild.base import fancylogger -from easybuild.base.exceptions import LoggedException -from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_error, print_msg, time_str_since +from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, time_str_since from easybuild.tools.config import ERROR, IGNORE, WARN, build_option from easybuild.tools.hooks import RUN_SHELL_CMD, load_hooks, run_hook from easybuild.tools.utilities import trace_msg @@ -151,9 +150,6 @@ def raise_run_shell_cmd_error(cmd, exit_code, work_dir, output, stderr): # 3) run_cmd_cache decorator # 4) actual caller site frameinfo = inspect.getouterframes(inspect.currentframe())[3] - caller_file_name = frameinfo.filename - caller_line_nr = frameinfo.lineno - caller_function_name = frameinfo.function caller_info = (frameinfo.filename, frameinfo.lineno, frameinfo.function) raise RunShellCmdError(cmd, exit_code, work_dir, output, stderr, caller_info) @@ -290,7 +286,7 @@ def to_cmd_str(cmd): # return output as a regular string rather than a byte sequence (and non-UTF-8 characters get stripped out) output = proc.stdout.decode('utf-8', 'ignore') - stderr= proc.stderr.decode('utf-8', 'ignore') if split_stderr else None + stderr = proc.stderr.decode('utf-8', 'ignore') if split_stderr else None res = RunShellCmdResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr, work_dir=work_dir) From eb768a1f54de6081bdf8a20e2cf1b56f03cf7380 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Oct 2023 19:52:57 +0200 Subject: [PATCH 145/312] fix broken tests due to changes in error handling --- test/framework/run.py | 7 ++++--- test/framework/toy_build.py | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/test/framework/run.py b/test/framework/run.py index 0e4de57cf7..529dbb7de1 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -249,7 +249,7 @@ def test_run_shell_cmd_log(self): os.close(fd) regex_start_cmd = re.compile("Running command 'echo hello' in /") - regex_cmd_exit = re.compile("Command 'echo hello' exited with exit code [0-9]* and output:") + regex_cmd_exit = re.compile("Shell command completed successfully \(see output above\): echo hello") # command output is always logged init_logging(logfile, silent=True) @@ -258,8 +258,9 @@ def test_run_shell_cmd_log(self): stop_logging(logfile) self.assertEqual(res.exit_code, 0) self.assertEqual(res.output, 'hello\n') - self.assertEqual(len(regex_start_cmd.findall(read_file(logfile))), 1) - self.assertEqual(len(regex_cmd_exit.findall(read_file(logfile))), 1) + logtxt = read_file(logfile) + self.assertEqual(len(regex_start_cmd.findall(logtxt)), 1) + self.assertEqual(len(regex_cmd_exit.findall(logtxt)), 1) write_file(logfile, '') # with debugging enabled, exit code and output of command should only get logged once diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 5b83b98a62..d77c9ebf11 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -374,8 +374,8 @@ def test_toy_buggy_easyblock(self): 'verify': False, 'verbose': False, } - err_regex = r"Traceback[\S\s]*toy_buggy.py.*build_step[\S\s]*name 'run_cmd' is not defined" - self.assertErrorRegex(EasyBuildError, err_regex, self.run_test_toy_build_with_output, **kwargs) + err_regex = r"name 'run_cmd' is not defined" + self.assertErrorRegex(NameError, err_regex, self.run_test_toy_build_with_output, **kwargs) def test_toy_build_formatv2(self): """Perform a toy build (format v2).""" @@ -1435,7 +1435,7 @@ def test_toy_extension_extract_cmd(self): ]) write_file(test_ec, test_ec_txt) - error_pattern = "unzip .*/bar-0.0.tar.gz.* returned non-zero exit status" + error_pattern = r"shell command 'unzip \.\.\.' failed in extensions step for test.eb" with self.mocked_stdout_stderr(): # for now, we expect subprocess.CalledProcessError, but eventually 'run' function will # do proper error reporting @@ -2642,7 +2642,7 @@ def test_toy_build_enhanced_sanity_check(self): test_ec_txt = test_ec_txt + '\nenhance_sanity_check = False' write_file(test_ec, test_ec_txt) - error_pattern = r" Missing mandatory key 'dirs' in sanity_check_paths." + error_pattern = r"Missing mandatory key 'dirs' in sanity_check_paths." with self.mocked_stdout_stderr(): self.assertErrorRegex(EasyBuildError, error_pattern, self._test_toy_build, ec_file=test_ec, extra_args=eb_args, raise_error=True, verbose=False) From 37d7a0edef8fba359221aa026b45acba8f530db1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Oct 2023 19:55:15 +0200 Subject: [PATCH 146/312] remove CI workflow for running framework tests with Python 2 --- .github/workflows/unit_tests_python2.yml | 81 ------------------------ 1 file changed, 81 deletions(-) delete mode 100644 .github/workflows/unit_tests_python2.yml diff --git a/.github/workflows/unit_tests_python2.yml b/.github/workflows/unit_tests_python2.yml deleted file mode 100644 index 1b921ee83c..0000000000 --- a/.github/workflows/unit_tests_python2.yml +++ /dev/null @@ -1,81 +0,0 @@ -# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions -name: EasyBuild framework unit tests (python2) -on: [push, pull_request] - -permissions: - contents: read # to fetch code (actions/checkout) - -concurrency: - group: ${{format('{0}:{1}:{2}', github.repository, github.ref, github.workflow)}} - cancel-in-progress: true - -jobs: - test_python2: - runs-on: ubuntu-20.04 - container: - # CentOS 7.9 container that already includes Lmod & co, - # see https://github.com/easybuilders/easybuild-containers - image: ghcr.io/easybuilders/centos-7.9-amd64 - steps: - - uses: actions/checkout@v3 - - - name: install Python packages - run: | - # Python packages - python2 -V - python2 -m pip --version - python2 -m pip install --upgrade pip - python2 -m pip --version - # strip out GC3Pie since installation with ancient setuptools (0.9.8) fails - sed -i '/GC3Pie/d' requirements.txt - python2 -m pip install -r requirements.txt - # git config is required to make actual git commits (cfr. tests for GitRepository) - sudo -u easybuild git config --global user.name "GitHub Actions" - sudo -u easybuild git config --global user.email "actions@github.com" - sudo -u easybuild git config --get-regexp 'user.*' - - - name: install GitHub token (if available) - env: - # token (owned by @boegelbot) with gist permissions (required for some of the tests for GitHub integration); - # this token is not available in pull requests, so tests that require it are skipped in PRs, - # and are only run after the PR gets merged - GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}} - run: | - # tests that require a GitHub token are skipped automatically when no GitHub token is available - if [ ! -z $GITHUB_TOKEN ]; then - sudo -u easybuild python2 -c "import keyring; import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring()); keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; - echo "GitHub token installed!" - else - echo "Installation of GitHub token skipped!" - fi - - - name: install sources - run: | - # install from source distribution tarball, to test release as published on PyPI - python2 setup.py sdist - ls dist - export PREFIX=/tmp/$USER/$GITHUB_SHA - python2 -m pip install --prefix $PREFIX dist/easybuild-framework*tar.gz - - - name: run test suite - run: | - # run tests *outside* of checked out easybuild-framework directory, - # to ensure we're testing installed version (see previous step) - cd $HOME - # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that) - export PREFIX=/tmp/$USER/$GITHUB_SHA - ENV_CMDS="export PATH=$PREFIX/bin:$PATH; export PYTHONPATH=$PREFIX/lib/python2.7/site-packages:$PYTHONPATH" - ENV_CMDS="${ENV_CMDS}; export EB_VERBOSE=1; export EB_PYTHON=python2; export TEST_EASYBUILD_SILENCE_DEPRECATION_WARNINGS=python2" - # run EasyBuild command via (non-root) easybuild user + login shell - sudo -u easybuild bash -l -c "${ENV_CMDS}; module --version; eb --version" - # show active EasyBuild configuration - sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --show-config" - # gather some useful info on test system - sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --show-system-info" - # check GitHub configuration - sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --check-github --github-user=easybuild_test" - # create file owned by root but writable by anyone (used by test_copy_file) - sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt - sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt - # run test suite (via easybuild user + login shell) - sudo -u easybuild bash -l -c "${ENV_CMDS}; python2 -O -m test.framework.suite" From 219be7ce1902b4ce4b55a821617c525f7b291f82 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Oct 2023 19:57:44 +0200 Subject: [PATCH 147/312] fix faulty imports in toy_build test module --- test/framework/run.py | 2 +- test/framework/toy_build.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/run.py b/test/framework/run.py index 529dbb7de1..3efa53debe 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -249,7 +249,7 @@ def test_run_shell_cmd_log(self): os.close(fd) regex_start_cmd = re.compile("Running command 'echo hello' in /") - regex_cmd_exit = re.compile("Shell command completed successfully \(see output above\): echo hello") + regex_cmd_exit = re.compile(r"Shell command completed successfully \(see output above\): echo hello") # command output is always logged init_logging(logfile, silent=True) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index d77c9ebf11..d2291d9625 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -58,7 +58,7 @@ from easybuild.tools.filetools import read_file, remove_dir, remove_file, which, write_file from easybuild.tools.module_generator import ModuleGeneratorTcl from easybuild.tools.modules import Lmod -from easybuild.tools.run import RunShellCmdError, run_cmd, run_shell_cmd +from easybuild.tools.run import run_cmd from easybuild.tools.utilities import nub from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.version import VERSION as EASYBUILD_VERSION From 9ab36ed097f540195cc7241d10404d45e1130fd5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Oct 2023 20:12:36 +0200 Subject: [PATCH 148/312] don't use f-strings without a placeholder --- easybuild/tools/run.py | 2 +- test/framework/toy_build.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 9ab09ade64..90699e8a96 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -106,7 +106,7 @@ def pad_4_spaces(msg): cmd_name = err.cmd.split(' ')[0] error_info = [ '', - f"ERROR: Shell command failed!", + "ERROR: Shell command failed!", pad_4_spaces(f"full command -> {err.cmd}"), pad_4_spaces(f"exit code -> {err.exit_code}"), pad_4_spaces(f"working directory -> {err.work_dir}"), diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index d2291d9625..74a467175b 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -4087,7 +4087,7 @@ def pre_configure_hook(self, *args, **kwargs): args = [ toy_ec, f'--hooks={hooks_file}', - f'--force', + '--force', f'--installpath={self.test_prefix}', f'--include-easyblocks={toy_eb}', ] From 23778be7d7b0f4e73a069d076fd89c8fa0dd4156 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 12 Oct 2023 15:19:48 +0100 Subject: [PATCH 149/312] fix parallellisation typo --- easybuild/toolchains/compiler/gcc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index 7a2c3770e3..3fba8d3954 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -50,7 +50,7 @@ class Gcc(Compiler): COMPILER_FAMILY = TC_CONSTANT_GCC COMPILER_UNIQUE_OPTS = { - 'loop': (False, "Automatic loop parallellisation"), + 'loop': (False, "Automatic loop parallelisation"), 'f2c': (False, "Generate code compatible with f2c and f77"), 'lto': (False, "Enable Link Time Optimization"), } From 13370c1c7d6387747e7bcbeeb886acb54e18f428 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 12 Oct 2023 15:38:35 +0100 Subject: [PATCH 150/312] parallelism --- easybuild/tools/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index a55cea4bbf..3cc0d5b7fb 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -473,7 +473,7 @@ def override_options(self): 'output-style': ("Control output style; auto implies using Rich if available to produce rich output, " "with fallback to basic colored output", 'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES), - 'parallel': ("Specify (maximum) level of parallellism used during build procedure", + 'parallel': ("Specify (maximum) level of parallelism used during build procedure", 'int', 'store', None), 'parallel-extensions-install': ("Install list of extensions in parallel (if supported)", None, 'store_true', False), From c2a8c55bd7ad2aca0aba8c847734dd588e8e3540 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 12 Oct 2023 15:38:54 +0100 Subject: [PATCH 151/312] parallelism --- easybuild/tools/systemtools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index b81e018bba..78ba312203 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -1212,7 +1212,7 @@ def get_default_parallelism(): raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err) if maxpar is not None and maxpar < par: - _log.info("Limiting parallellism from %s to %s", par, maxpar) + _log.info("Limiting parallelism from %s to %s", par, maxpar) par = maxpar return par From 5370a6697b58403a0979a4740c1f83e5dacd27e6 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 12 Oct 2023 15:39:28 +0100 Subject: [PATCH 152/312] parallelism --- test/framework/systemtools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 9186b750ec..1707d6ec54 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -835,9 +835,9 @@ def test_system_info(self): def test_det_parallelism_native(self): """Test det_parallelism function (native calls).""" self.assertTrue(det_parallelism() > 0) - # specified parallellism + # specified parallelism self.assertEqual(det_parallelism(par=5), 5) - # max parallellism caps + # max parallelism caps self.assertEqual(det_parallelism(maxpar=1), 1) self.assertEqual(det_parallelism(16, 1), 1) self.assertEqual(det_parallelism(par=5, maxpar=2), 2) From dcf9bb3785f34d8781379875fb2e5b0334e4c29a Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 12 Oct 2023 15:39:53 +0100 Subject: [PATCH 153/312] parallelism --- test/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 819d74fc7a..5e2407575d 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -2080,7 +2080,7 @@ def test_extensions_sanity_check(self): eb.run_all_steps(True) def test_parallel(self): - """Test defining of parallellism.""" + """Test defining of parallelism.""" topdir = os.path.abspath(os.path.dirname(__file__)) toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') toytxt = read_file(toy_ec) @@ -2097,7 +2097,7 @@ def test_parallel(self): os.close(handle) write_file(toy_ec3, toytxt + "\nparallel = False") - # default: parallellism is derived from # available cores + ulimit + # default: parallelism is derived from # available cores + ulimit test_eb = EasyBlock(EasyConfig(toy_ec)) test_eb.check_readiness_step() self.assertTrue(isinstance(test_eb.cfg['parallel'], int) and test_eb.cfg['parallel'] > 0) From ece71c2b1cfe2264fad792269aab9d16dfe1c7f6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 16 Oct 2023 09:32:30 +0200 Subject: [PATCH 154/312] collapse print_run_shell_cmd_error into RunShellCmdError as print method --- easybuild/framework/easyblock.py | 4 +- easybuild/tools/run.py | 68 ++++++++++++++++---------------- test/framework/run.py | 6 +-- 3 files changed, 38 insertions(+), 40 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 79d32ff20a..5886fe8500 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -87,7 +87,7 @@ from easybuild.tools.hooks import MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP from easybuild.tools.hooks import POSTPROC_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP from easybuild.tools.hooks import SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook -from easybuild.tools.run import RunShellCmdError, check_async_cmd, print_run_shell_cmd_error, run_cmd +from easybuild.tools.run import RunShellCmdError, check_async_cmd, run_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version @@ -4125,7 +4125,7 @@ def run_all_steps(self, run_test_cases): try: self.run_step(step_name, step_methods) except RunShellCmdError as err: - print_run_shell_cmd_error(err) + err.print() ec_path = os.path.basename(self.cfg.path) error_msg = f"shell command '{err.cmd_name} ...' failed in {step_name} step for {ec_path}" raise EasyBuildError(error_msg) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 90699e8a96..e43edbe7cf 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -94,48 +94,46 @@ def __init__(self, cmd, exit_code, work_dir, output, stderr, caller_info, *args, msg = f"Shell command '{self.cmd_name}' failed!" super(RunShellCmdError, self).__init__(msg, *args, **kwargs) + def print(self): + """ + Report failed shell command for this RunShellCmdError instance + """ -def print_run_shell_cmd_error(err): - """ - Report failed shell command using provided RunShellCmdError instance - """ + def pad_4_spaces(msg): + return ' ' * 4 + msg - def pad_4_spaces(msg): - return ' ' * 4 + msg + error_info = [ + '', + "ERROR: Shell command failed!", + pad_4_spaces(f"full command -> {self.cmd}"), + pad_4_spaces(f"exit code -> {self.exit_code}"), + pad_4_spaces(f"working directory -> {self.work_dir}"), + ] - cmd_name = err.cmd.split(' ')[0] - error_info = [ - '', - "ERROR: Shell command failed!", - pad_4_spaces(f"full command -> {err.cmd}"), - pad_4_spaces(f"exit code -> {err.exit_code}"), - pad_4_spaces(f"working directory -> {err.work_dir}"), - ] + tmpdir = tempfile.mkdtemp(prefix='shell-cmd-error-') + output_fp = os.path.join(tmpdir, f"{self.cmd_name}.out") + with open(output_fp, 'w') as fp: + fp.write(self.output or '') - tmpdir = tempfile.mkdtemp(prefix='shell-cmd-error-') - output_fp = os.path.join(tmpdir, f"{cmd_name}.out") - with open(output_fp, 'w') as fp: - fp.write(err.output or '') - - if err.stderr is None: - error_info.append(pad_4_spaces(f"output (stdout + stderr) -> {output_fp}")) - else: - stderr_fp = os.path.join(tmpdir, f"{cmd_name}.err") - with open(stderr_fp, 'w') as fp: - fp.write(err.stderr) + if self.stderr is None: + error_info.append(pad_4_spaces(f"output (stdout + stderr) -> {output_fp}")) + else: + stderr_fp = os.path.join(tmpdir, f"{self.cmd_name}.err") + with open(stderr_fp, 'w') as fp: + fp.write(self.stderr) + error_info.extend([ + pad_4_spaces(f"output (stdout) -> {output_fp}"), + pad_4_spaces(f"error/warnings (stderr) -> {stderr_fp}"), + ]) + + caller_file_name, caller_line_nr, caller_function_name = self.caller_info + called_from_info = f"'{caller_function_name}' function in {caller_file_name} (line {caller_line_nr})" error_info.extend([ - pad_4_spaces(f"output (stdout) -> {output_fp}"), - pad_4_spaces(f"error/warnings (stderr) -> {stderr_fp}"), + pad_4_spaces(f"called from -> {called_from_info}"), + '', ]) - caller_file_name, caller_line_nr, caller_function_name = err.caller_info - called_from_info = f"'{caller_function_name}' function in {caller_file_name} (line {caller_line_nr})" - error_info.extend([ - pad_4_spaces(f"called from -> {called_from_info}"), - '', - ]) - - sys.stderr.write('\n'.join(error_info) + '\n') + sys.stderr.write('\n'.join(error_info) + '\n') def raise_run_shell_cmd_error(cmd, exit_code, work_dir, output, stderr): diff --git a/test/framework/run.py b/test/framework/run.py index 3efa53debe..5bce2d20f1 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -52,7 +52,7 @@ from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir, read_file, write_file from easybuild.tools.run import RunShellCmdResult, RunShellCmdError, check_async_cmd, check_log_for_errors from easybuild.tools.run import complete_cmd, get_output_from_process, parse_log_for_error -from easybuild.tools.run import print_run_shell_cmd_error, run_cmd, run_cmd_qa, run_shell_cmd, subprocess_terminate +from easybuild.tools.run import run_cmd, run_cmd_qa, run_shell_cmd, subprocess_terminate from easybuild.tools.config import ERROR, IGNORE, WARN @@ -346,7 +346,7 @@ def handler(signum, _): self.assertEqual(err.caller_info[2], 'test_run_shell_cmd_fail') with self.mocked_stdout_stderr() as (_, stderr): - print_run_shell_cmd_error(err) + err.print() # check error reporting output stderr = stderr.getvalue() @@ -381,7 +381,7 @@ def handler(signum, _): self.assertEqual(err.caller_info[2], 'test_run_shell_cmd_fail') with self.mocked_stdout_stderr() as (_, stderr): - print_run_shell_cmd_error(err) + err.print() # check error reporting output stderr = stderr.getvalue() From f3eafe67bafadd80c53ef9433d5e37c7288070d8 Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 17 Oct 2023 08:57:08 +0200 Subject: [PATCH 155/312] minor help fix --- easybuild/tools/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index a55cea4bbf..61d75cb656 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -276,8 +276,8 @@ def basic_options(self): 'only-blocks': ("Only build listed blocks", 'strlist', 'extend', None, 'b', {'metavar': 'BLOCKS'}), 'rebuild': ("Rebuild software, even if module already exists (don't skip OS dependencies checks)", None, 'store_true', False), - 'robot': ("Enable dependency resolution, using easyconfigs in specified paths", - 'pathlist', 'store_or_None', [], 'r', {'metavar': 'PATH[%sPATH]' % os.pathsep}), + 'robot': ("Enable dependency resolution, optionally consider additional paths to search for easyconfigs", + 'pathlist', 'store_or_None', [], 'r', {'metavar': '[PATH[%sPATH]]' % os.pathsep}), 'robot-paths': ("Additional paths to consider by robot for easyconfigs (--robot paths get priority)", 'pathlist', 'add_flex', self.default_robot_paths, {'metavar': 'PATH[%sPATH]' % os.pathsep}), 'search-paths': ("Additional locations to consider in --search (next to --robot and --robot-paths paths)", From 3daac777c75c63f9df77e68bb6752e32444bc895 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 18 Oct 2023 21:04:25 +0200 Subject: [PATCH 156/312] always store command output to temporary file in run_shell_cmd --- easybuild/tools/run.py | 91 ++++++++++++++++++++++-------------------- test/framework/run.py | 35 ++++++++-------- 2 files changed, 66 insertions(+), 60 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index e43edbe7cf..c8ec8373c7 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -76,19 +76,16 @@ ] -RunShellCmdResult = namedtuple('RunShellCmdResult', ('cmd', 'exit_code', 'output', 'stderr', 'work_dir')) +RunShellCmdResult = namedtuple('RunShellCmdResult', ('cmd', 'exit_code', 'output', 'stderr', 'work_dir', + 'out_file', 'err_file')) class RunShellCmdError(BaseException): - def __init__(self, cmd, exit_code, work_dir, output, stderr, caller_info, *args, **kwargs): + def __init__(self, cmd_result, caller_info, *args, **kwargs): """Constructor for RunShellCmdError.""" - self.cmd = cmd - self.cmd_name = cmd.split(' ')[0] - self.exit_code = exit_code - self.work_dir = work_dir - self.output = output - self.stderr = stderr + self.cmd_result = cmd_result + self.cmd_name = cmd_result.cmd.split(' ')[0] self.caller_info = caller_info msg = f"Shell command '{self.cmd_name}' failed!" @@ -105,25 +102,17 @@ def pad_4_spaces(msg): error_info = [ '', "ERROR: Shell command failed!", - pad_4_spaces(f"full command -> {self.cmd}"), - pad_4_spaces(f"exit code -> {self.exit_code}"), - pad_4_spaces(f"working directory -> {self.work_dir}"), + pad_4_spaces(f"full command -> {self.cmd_result.cmd}"), + pad_4_spaces(f"exit code -> {self.cmd_result.exit_code}"), + pad_4_spaces(f"working directory -> {self.cmd_result.work_dir}"), ] - tmpdir = tempfile.mkdtemp(prefix='shell-cmd-error-') - output_fp = os.path.join(tmpdir, f"{self.cmd_name}.out") - with open(output_fp, 'w') as fp: - fp.write(self.output or '') - - if self.stderr is None: - error_info.append(pad_4_spaces(f"output (stdout + stderr) -> {output_fp}")) + if self.cmd_result.stderr is None: + error_info.append(pad_4_spaces(f"output (stdout + stderr) -> {self.cmd_result.out_file}")) else: - stderr_fp = os.path.join(tmpdir, f"{self.cmd_name}.err") - with open(stderr_fp, 'w') as fp: - fp.write(self.stderr) error_info.extend([ - pad_4_spaces(f"output (stdout) -> {output_fp}"), - pad_4_spaces(f"error/warnings (stderr) -> {stderr_fp}"), + pad_4_spaces(f"output (stdout) -> {self.cmd_result.out_file}"), + pad_4_spaces(f"error/warnings (stderr) -> {self.cmd_result.err_file}"), ]) caller_file_name, caller_line_nr, caller_function_name = self.caller_info @@ -136,9 +125,9 @@ def pad_4_spaces(msg): sys.stderr.write('\n'.join(error_info) + '\n') -def raise_run_shell_cmd_error(cmd, exit_code, work_dir, output, stderr): +def raise_run_shell_cmd_error(cmd_res): """ - Raise RunShellCmdError for failing shell command, after collecting additional caller info + Raise RunShellCmdError for failed shell command, after collecting additional caller info """ # figure out where failing command was run @@ -150,7 +139,7 @@ def raise_run_shell_cmd_error(cmd, exit_code, work_dir, output, stderr): frameinfo = inspect.getouterframes(inspect.currentframe())[3] caller_info = (frameinfo.filename, frameinfo.lineno, frameinfo.function) - raise RunShellCmdError(cmd, exit_code, work_dir, output, stderr, caller_info) + raise RunShellCmdError(cmd_res, caller_info) def run_cmd_cache(func): @@ -234,16 +223,18 @@ def to_cmd_str(cmd): if work_dir is None: work_dir = os.getcwd() - # temporary output file for command output, if requested - if output_file or not hidden: - # collect output of running command in temporary log file, if desired - fd, cmd_out_fp = tempfile.mkstemp(suffix='.log', prefix='easybuild-run-') - os.close(fd) - _log.info(f'run_cmd: Output of "{cmd}" will be logged to {cmd_out_fp}') - else: - cmd_out_fp = None - cmd_str = to_cmd_str(cmd) + cmd_name = cmd_str.split(' ')[0] + + # temporary output file(s) for command output + tmpdir = tempfile.mkdtemp(prefix='run-shell-cmd-') + cmd_out_fp = os.path.join(tmpdir, f'{cmd_name}.out') + _log.info(f'run_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}') + if split_stderr: + cmd_err_fp = os.path.join(tmpdir, f'{cmd_name}.err') + _log.info(f'run_cmd: Errors and warnings of "{cmd_str}" will be logged to {cmd_err_fp}') + else: + cmd_err_fp = None # early exit in 'dry run' mode, after printing the command that would be run (unless 'hidden' is enabled) if not in_dry_run and build_option('extended_dry_run'): @@ -253,11 +244,12 @@ def to_cmd_str(cmd): msg += f" (in {work_dir})" dry_run_msg(msg, silent=silent) - return RunShellCmdResult(cmd=cmd_str, exit_code=0, output='', stderr=None, work_dir=work_dir) + return RunShellCmdResult(cmd=cmd_str, exit_code=0, output='', stderr=None, work_dir=work_dir, + out_file=cmd_out_fp, err_file=cmd_err_fp) start_time = datetime.now() if not hidden: - cmd_trace_msg(cmd_str, start_time, work_dir, stdin, cmd_out_fp) + cmd_trace_msg(cmd_str, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp) if stdin: # 'input' value fed to subprocess.run must be a byte sequence @@ -286,7 +278,18 @@ def to_cmd_str(cmd): output = proc.stdout.decode('utf-8', 'ignore') stderr = proc.stderr.decode('utf-8', 'ignore') if split_stderr else None - res = RunShellCmdResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr, work_dir=work_dir) + # store command output to temporary file(s) + try: + with open(cmd_out_fp, 'w') as fp: + fp.write(output) + if split_stderr: + with open(cmd_err_fp, 'w') as fp: + fp.write(stderr) + except IOError as err: + raise EasyBuildError(f"Failed to dump command output to temporary file: {err}") + + res = RunShellCmdResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr, work_dir=work_dir, + out_file=cmd_out_fp, err_file=cmd_err_fp) # always log command output cmd_name = cmd_str.split(' ')[0] @@ -301,7 +304,7 @@ def to_cmd_str(cmd): else: _log.warning(f"Shell command FAILED (exit code {res.exit_code}, see output above): {cmd_str}") if fail_on_error: - raise_run_shell_cmd_error(res.cmd, res.exit_code, res.work_dir, output=res.output, stderr=res.stderr) + raise_run_shell_cmd_error(res) if with_hooks: run_hook_kwargs = { @@ -319,7 +322,7 @@ def to_cmd_str(cmd): return res -def cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp): +def cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp): """ Helper function to construct and print trace message for command being run @@ -327,7 +330,8 @@ def cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp): :param start_time: datetime object indicating when command was started :param work_dir: path of working directory in which command is run :param stdin: stdin input value for command - :param cmd_out_fp: path to output log file for command + :param cmd_out_fp: path to output file for command + :param cmd_err_fp: path to errors/warnings output file for command """ start_time = start_time.strftime('%Y-%m-%d %H:%M:%S') @@ -338,8 +342,9 @@ def cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp): ] if stdin: lines.append(f"\t[input: {stdin}]") - if cmd_out_fp: - lines.append(f"\t[output logged in {cmd_out_fp}]") + lines.append(f"\t[output saved to {cmd_out_fp}]") + if cmd_err_fp: + lines.append(f"\t[errors/warnings saved to {cmd_err_fp}]") lines.append('\t' + cmd) diff --git a/test/framework/run.py b/test/framework/run.py index 5bce2d20f1..91b03d1b44 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -333,12 +333,12 @@ def handler(signum, _): self.assertFalse("This should never be reached, RunShellCmdError should occur!") except RunShellCmdError as err: self.assertEqual(str(err), "Shell command 'kill' failed!") - self.assertEqual(err.cmd, "kill -9 $$") + self.assertEqual(err.cmd_result.cmd, "kill -9 $$") self.assertEqual(err.cmd_name, 'kill') - self.assertEqual(err.exit_code, -9) - self.assertEqual(err.work_dir, work_dir) - self.assertEqual(err.output, '') - self.assertEqual(err.stderr, None) + self.assertEqual(err.cmd_result.exit_code, -9) + self.assertEqual(err.cmd_result.work_dir, work_dir) + self.assertEqual(err.cmd_result.output, '') + self.assertEqual(err.cmd_result.stderr, None) self.assertTrue(isinstance(err.caller_info, tuple)) self.assertEqual(len(err.caller_info), 3) self.assertEqual(err.caller_info[0], __file__) @@ -356,7 +356,7 @@ def handler(signum, _): r"^\s+exit code\s* -> -9", r"^\s+working directory\s* -> " + work_dir, r"^\s+called from\s* -> 'test_run_shell_cmd_fail' function in .*/test/.*/run.py \(line [0-9]+\)", - r"^\s+output \(stdout \+ stderr\)\s* -> .*/shell-cmd-error-.*/kill.out", + r"^\s+output \(stdout \+ stderr\)\s* -> .*/run-shell-cmd-.*/kill.out", ] for pattern in patterns: regex = re.compile(pattern, re.M) @@ -368,12 +368,12 @@ def handler(signum, _): self.assertFalse("This should never be reached, RunShellCmdError should occur!") except RunShellCmdError as err: self.assertEqual(str(err), "Shell command 'kill' failed!") - self.assertEqual(err.cmd, "kill -9 $$") + self.assertEqual(err.cmd_result.cmd, "kill -9 $$") self.assertEqual(err.cmd_name, 'kill') - self.assertEqual(err.exit_code, -9) - self.assertEqual(err.work_dir, work_dir) - self.assertEqual(err.output, '') - self.assertEqual(err.stderr, '') + self.assertEqual(err.cmd_result.exit_code, -9) + self.assertEqual(err.cmd_result.work_dir, work_dir) + self.assertEqual(err.cmd_result.output, '') + self.assertEqual(err.cmd_result.stderr, '') self.assertTrue(isinstance(err.caller_info, tuple)) self.assertEqual(len(err.caller_info), 3) self.assertEqual(err.caller_info[0], __file__) @@ -391,8 +391,8 @@ def handler(signum, _): r"^\s+exit code\s+ -> -9", r"^\s+working directory\s+ -> " + work_dir, r"^\s+called from\s+ -> 'test_run_shell_cmd_fail' function in .*/test/.*/run.py \(line [0-9]+\)", - r"^\s+output \(stdout\)\s+ -> .*/shell-cmd-error-.*/kill.out", - r"^\s+error/warnings \(stderr\)\s+ -> .*/shell-cmd-error-.*/kill.err", + r"^\s+output \(stdout\)\s+ -> .*/run-shell-cmd-.*/kill.out", + r"^\s+error/warnings \(stderr\)\s+ -> .*/run-shell-cmd-.*/kill.err", ] for pattern in patterns: regex = re.compile(pattern, re.M) @@ -618,7 +618,7 @@ def test_run_shell_cmd_trace(self): r"^ >> running command:", r"\t\[started at: .*\]", r"\t\[working dir: .*\]", - r"\t\[output logged in .*\]", + r"\t\[output saved to .*\]", r"\techo hello", r" >> command completed: exit 0, ran in .*", ] @@ -678,7 +678,7 @@ def test_run_shell_cmd_trace_stdin(self): r"^ >> running command:", r"\t\[started at: [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]\]", r"\t\[working dir: .*\]", - r"\t\[output logged in .*\]", + r"\t\[output saved to .*\]", r"\techo hello", r" >> command completed: exit 0, ran in .*", ] @@ -909,7 +909,7 @@ def test_run_shell_cmd_cache(self): # inject value into cache to check whether executing command again really returns cached value with self.mocked_stdout_stderr(): cached_res = RunShellCmdResult(cmd=cmd, output="123456", exit_code=123, stderr=None, - work_dir='/test_ulimit') + work_dir='/test_ulimit', out_file='/tmp/foo.out', err_file=None) run_shell_cmd.update_cache({(cmd, None): cached_res}) res = run_shell_cmd(cmd) self.assertEqual(res.cmd, cmd) @@ -927,7 +927,8 @@ def test_run_shell_cmd_cache(self): # inject different output for cat with 'foo' as stdin to check whether cached value is used with self.mocked_stdout_stderr(): - cached_res = RunShellCmdResult(cmd=cmd, output="bar", exit_code=123, stderr=None, work_dir='/test_cat') + cached_res = RunShellCmdResult(cmd=cmd, output="bar", exit_code=123, stderr=None, + work_dir='/test_cat', out_file='/tmp/cat.out', err_file=None) run_shell_cmd.update_cache({(cmd, 'foo'): cached_res}) res = run_shell_cmd(cmd, stdin='foo') self.assertEqual(res.cmd, cmd) From 7ee39c55baedcfed230cc3ef76f41422d8650aaf Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 22 Oct 2023 12:57:21 +0200 Subject: [PATCH 157/312] take into account that VERBOSE_VERSION imported from easybuild.easyblocks is now a string value --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index b5e82834cc..2ea349637d 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -92,7 +92,7 @@ def get_git_revision(): def this_is_easybuild(): """Standard starting message""" - top_version = max(FRAMEWORK_VERSION, EASYBLOCKS_VERSION) + top_version = max(FRAMEWORK_VERSION, LooseVersion(EASYBLOCKS_VERSION)) msg = "This is EasyBuild %s (framework: %s, easyblocks: %s) on host %s." msg = msg % (top_version, FRAMEWORK_VERSION, EASYBLOCKS_VERSION, gethostname()) From 62b4c165a8d634a6578ff4e10e83a0457ea96a18 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 23 Oct 2023 10:33:40 +0200 Subject: [PATCH 158/312] use 5.0.x branch for easyblocks + easyconfigs in CI workflows (for now) --- .github/workflows/container_tests.yml | 4 ++-- .github/workflows/container_tests_apptainer.yml | 4 ++-- .github/workflows/end2end.yml | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/container_tests.yml b/.github/workflows/container_tests.yml index ef310a3816..41e6820dd8 100644 --- a/.github/workflows/container_tests.yml +++ b/.github/workflows/container_tests.yml @@ -74,7 +74,7 @@ jobs: ls dist export PREFIX=/tmp/$USER/$GITHUB_SHA pip install --prefix $PREFIX dist/easybuild-framework*tar.gz - pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/develop.tar.gz + pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/5.0.x.tar.gz - name: run test run: | @@ -95,7 +95,7 @@ jobs: echo '%_dbpath %{_var}/lib/rpm' >> $HOME/.rpmmacros # build CentOS 7 container image for bzip2 1.0.8 using EasyBuild; # see https://docs.easybuild.io/en/latest/Containers.html - curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb + curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/5.0.x/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb export EASYBUILD_CONTAINERPATH=$PWD export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/centos-7.9-python3-amd64' eb bzip2-1.0.8.eb --containerize --experimental --container-build-image diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml index 35c26c26c9..1f46f2f5f6 100644 --- a/.github/workflows/container_tests_apptainer.yml +++ b/.github/workflows/container_tests_apptainer.yml @@ -66,7 +66,7 @@ jobs: ls dist export PREFIX=/tmp/$USER/$GITHUB_SHA pip install --prefix $PREFIX dist/easybuild-framework*tar.gz - pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/develop.tar.gz + pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/5.0.x.tar.gz - name: run test run: | @@ -87,7 +87,7 @@ jobs: echo '%_dbpath %{_var}/lib/rpm' >> $HOME/.rpmmacros # build CentOS 7 container image for bzip2 1.0.8 using EasyBuild; # see https://docs.easybuild.io/en/latest/Containers.html - curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb + curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/5.0.x/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb export EASYBUILD_CONTAINERPATH=$PWD export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/centos-7.9-python3-amd64' export EASYBUILD_CONTAINER_TYPE='apptainer' diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index dae567d575..fc4936286a 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -25,9 +25,9 @@ jobs: run: | cd $HOME for pkg in easyblocks easyconfigs; do - curl -OL https://github.com/easybuilders/easybuild-${pkg}/archive/develop.tar.gz - tar xfz develop.tar.gz - rm -f develop.tar.gz + curl -OL https://github.com/easybuilders/easybuild-${pkg}/archive/5.0.x.tar.gz + tar xfz 5.0.x.tar.gz + rm -f 5.0.x.tar.gz done - name: Set up environment @@ -35,7 +35,7 @@ jobs: run: | # collect environment variables to be set in subsequent steps in script that can be sourced echo "export PATH=$PWD:$PATH" > /tmp/eb_env - echo "export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" >> /tmp/eb_env + echo "export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-5.0.x:$HOME/easybuild-easyconfigs-5.0.x" >> /tmp/eb_env - name: Run commands to check test environment shell: bash From 9d43ede93713450d155c1be774cc940f76e0ca0b Mon Sep 17 00:00:00 2001 From: Xin An Date: Tue, 24 Oct 2023 16:17:19 +0200 Subject: [PATCH 159/312] add sysroot template value --- easybuild/framework/easyconfig/templates.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 465028c3ca..fa8c8861ea 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -90,6 +90,8 @@ # template values which are only generated dynamically TEMPLATE_NAMES_DYNAMIC = [ ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"), + ('sysroot', "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include" + "as specify by the --sysroot configuration option"), ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"), ('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via " "--cuda-compute-capabilities configuration option or via cuda_compute_capabilities easyconfig parameter"), @@ -201,6 +203,13 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) # set 'arch' for system architecture based on 'machine' (4th) element of platform.uname() return value template_values['arch'] = platform.uname()[4] + # set 'sysroot' based on sysroot build option + sysroot = build_option('sysroot') + if sysroot == None: + template_values['sysroot'] = "" + else: + template_values['sysroot'] = sysroot + # step 1: add TEMPLATE_NAMES_EASYCONFIG for name in TEMPLATE_NAMES_EASYCONFIG: if name in ignore: From dfe0c8ed679a163f454f93bde967cbb1031a659a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 24 Oct 2023 17:14:35 +0200 Subject: [PATCH 160/312] Added unit tests: one for empty sysroot, one for non-empty sysroot --- test/framework/easyconfig.py | 70 ++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 14ed69084b..bd02de939d 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1346,6 +1346,76 @@ def test_start_dir_template(self): self.assertIn('start_dir in extension configure is %s &&' % ext_start_dir, logtxt) self.assertIn('start_dir in extension build is %s &&' % ext_start_dir, logtxt) + def test_sysroot_template_empty(self): + """Test the %(sysroot)s template whenever --sysroot is unset (i.e. None)""" + + self.contents = textwrap.dedent(""" + name = 'toy' + version = '0.0' + + homepage = 'https://easybuilders.github.io/easybuild' + description = 'Toy C program, 100% toy.' + + toolchain = SYSTEM + + sources = [SOURCE_TAR_GZ] + + preconfigopts = 'echo sysroot in configure is %(sysroot)s && ' + prebuildopts = 'echo sysroot in build is %(sysroot)s && ' + preinstallopts = 'echo sysroot in install is %(sysroot)s && ' + + moduleclass = 'tools' + """) + self.prep() + ec = EasyConfig(self.eb_file) + from easybuild.easyblocks.toy import EB_toy + eb = EB_toy(ec) + # Check behaviour when sysroot is not set (i.e. None) + eb.cfg['sysroot'] = None # Should we define this explicitely? Or rely on this to be the default? + eb.cfg['stop'] = 'extensions' + with self.mocked_stdout_stderr(): + eb.run_all_steps(False) + logtxt = read_file(eb.logfile) + sysroot = "" + self.assertIn('sysroot in configure is %s/ &&' % sysroot, logtxt) + self.assertIn('sysroot in build is %s/ &&' % sysroot, logtxt) + self.assertIn('sysroot in install is %s/ &&' % sysroot, logtxt) + + def test_sysroot_template_non_empty(self): + """Test the %(sysroot)s template whenever --sysroot is unset (i.e. None)""" + + self.contents = textwrap.dedent(""" + name = 'toy' + version = '0.0' + + homepage = 'https://easybuilders.github.io/easybuild' + description = 'Toy C program, 100% toy.' + + toolchain = SYSTEM + + sources = [SOURCE_TAR_GZ] + + preconfigopts = 'echo sysroot in configure is %(sysroot)s && ' + prebuildopts = 'echo sysroot in build is %(sysroot)s && ' + preinstallopts = 'echo sysroot in install is %(sysroot)s && ' + + moduleclass = 'tools' + """) + self.prep() + ec = EasyConfig(self.eb_file) + from easybuild.easyblocks.toy import EB_toy + eb = EB_toy(ec) + # Check behaviour when sysroot is not set (i.e. None) + eb.cfg['sysroot'] = '/tmp' # This should be a path that exists, otherwise EasyBuild complains + eb.cfg['stop'] = 'extensions' + with self.mocked_stdout_stderr(): + eb.run_all_steps(False) + logtxt = read_file(eb.logfile) + sysroot = "" + self.assertIn('sysroot in configure is %s/ &&' % sysroot, logtxt) + self.assertIn('sysroot in build is %s/ &&' % sysroot, logtxt) + self.assertIn('sysroot in install is %s/ &&' % sysroot, logtxt) + def test_constant_doc(self): """test constant documentation""" doc = avail_easyconfig_constants() From 6d6571a902c6f004bdc7bd840b696e028f179ecf Mon Sep 17 00:00:00 2001 From: Xin An <34663977+xinan1911@users.noreply.github.com> Date: Tue, 24 Oct 2023 17:28:41 +0200 Subject: [PATCH 161/312] Update templates.py for the style check --- easybuild/framework/easyconfig/templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index fa8c8861ea..9949955854 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -205,7 +205,7 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) # set 'sysroot' based on sysroot build option sysroot = build_option('sysroot') - if sysroot == None: + if sysroot is None: template_values['sysroot'] = "" else: template_values['sysroot'] = sysroot From 2e6bd7a3c4662456dc050916ee4ee06b0cc9b82e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Wed, 25 Oct 2023 11:01:53 +0200 Subject: [PATCH 162/312] Update easybuild/framework/easyconfig/templates.py Co-authored-by: Kenneth Hoste --- easybuild/framework/easyconfig/templates.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 9949955854..07d838eee9 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -203,12 +203,9 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) # set 'arch' for system architecture based on 'machine' (4th) element of platform.uname() return value template_values['arch'] = platform.uname()[4] - # set 'sysroot' based on sysroot build option - sysroot = build_option('sysroot') - if sysroot is None: - template_values['sysroot'] = "" - else: - template_values['sysroot'] = sysroot + # set 'sysroot' template based on 'sysroot' configuration option, using empty string as fallback + value + template_values['sysroot'] = build_option('sysroot') or '' # step 1: add TEMPLATE_NAMES_EASYCONFIG for name in TEMPLATE_NAMES_EASYCONFIG: From d52c7792ff881e9f43b33804e2f3400f847b9703 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Oct 2023 11:11:12 +0200 Subject: [PATCH 163/312] Removed stray 'value' in code --- easybuild/framework/easyconfig/templates.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 07d838eee9..aed8db6af4 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -204,7 +204,6 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) template_values['arch'] = platform.uname()[4] # set 'sysroot' template based on 'sysroot' configuration option, using empty string as fallback - value template_values['sysroot'] = build_option('sysroot') or '' # step 1: add TEMPLATE_NAMES_EASYCONFIG From 510cb33b3a7e5b05524db47a081b409069099b89 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 25 Oct 2023 11:20:49 +0200 Subject: [PATCH 164/312] add `EasyConfig.dependency_names` --- easybuild/framework/easyconfig/easyconfig.py | 9 +++++++++ test/framework/easyconfig.py | 19 +++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 55fe3c8578..ff4ed3562c 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1139,6 +1139,15 @@ def dependencies(self, build_only=False): return retained_deps + def dependency_names(self, build_only=False): + """ + Return a set of names of all (direct) dependencies after filtering. + Iterable builddependencies are flattened when not iterating. + + :param build_only: only return build dependencies, discard others + """ + return {dep['name'] for dep in self.dependencies(build_only=build_only) if dep['name']} + def builddependencies(self): """ Return a flat list of the parsed build dependencies diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 14ed69084b..5fde75938d 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -301,7 +301,9 @@ def test_dependency(self): self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3') self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3') + self.assertEqual(eb.dependency_names(), {'first', 'second', 'foo', 'bar'}) # same tests for builddependencies + self.assertEqual(eb.dependency_names(build_only=True), {'first', 'second'}) first = eb.builddependencies()[0] second = eb.builddependencies()[1] @@ -354,6 +356,7 @@ def test_false_dep_version(self): self.assertEqual(len(deps), 2) self.assertEqual(deps[0]['name'], 'second_build') self.assertEqual(deps[1]['name'], 'first') + self.assertEqual(eb.dependency_names(), {'first', 'second_build'}) # more realistic example: only filter dep for POWER self.contents = '\n'.join([ @@ -377,12 +380,14 @@ def test_false_dep_version(self): deps = eb.dependencies() self.assertEqual(len(deps), 1) self.assertEqual(deps[0]['name'], 'not_on_power') + self.assertEqual(eb.dependency_names(), {'not_on_power'}) # only power, dependency gets filtered st.get_cpu_architecture = lambda: POWER eb = EasyConfig(self.eb_file) deps = eb.dependencies() self.assertEqual(deps, []) + self.assertEqual(eb.dependency_names(), {}) def test_extra_options(self): """ extra_options should allow other variables to be stored """ @@ -1608,18 +1613,15 @@ def test_filter_deps(self): test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') ec_file = os.path.join(test_ecs_dir, 'f', 'foss', 'foss-2018a.eb') ec = EasyConfig(ec_file) - deps = sorted([dep['name'] for dep in ec.dependencies()]) - self.assertEqual(deps, ['FFTW', 'GCC', 'OpenBLAS', 'OpenMPI', 'ScaLAPACK']) + self.assertEqual(ec.dependency_names(), {'FFTW', 'GCC', 'OpenBLAS', 'OpenMPI', 'ScaLAPACK'}) # test filtering multiple deps init_config(build_options={'filter_deps': ['FFTW', 'ScaLAPACK']}) - deps = sorted([dep['name'] for dep in ec.dependencies()]) - self.assertEqual(deps, ['GCC', 'OpenBLAS', 'OpenMPI']) + self.assertEqual(ec.dependency_names(), {'GCC', 'OpenBLAS', 'OpenMPI'}) # test filtering of non-existing dep init_config(build_options={'filter_deps': ['zlib']}) - deps = sorted([dep['name'] for dep in ec.dependencies()]) - self.assertEqual(deps, ['FFTW', 'GCC', 'OpenBLAS', 'OpenMPI', 'ScaLAPACK']) + self.assertEqual(ec.dependency_names(), {'FFTW', 'GCC', 'OpenBLAS', 'OpenMPI', 'ScaLAPACK'}) # test parsing of value passed to --filter-deps opts = init_config(args=[]) @@ -1653,6 +1655,7 @@ def test_filter_deps(self): init_config(build_options=build_options) ec = EasyConfig(ec_file, validate=False) self.assertEqual(ec.dependencies(), []) + self.assertEqual(ec.dependency_names(), {}) def test_replaced_easyconfig_parameters(self): """Test handling of replaced easyconfig parameters.""" @@ -1841,6 +1844,9 @@ def test_external_dependencies(self): } self.assertEqual(deps[7]['external_module_metadata'], cray_netcdf_metadata) + # External module names are omitted + self.assertEqual(ec.dependency_names(), {'intel'}) + # provide file with partial metadata for some external modules; # metadata obtained from probing modules should be added to it... metadata = os.path.join(self.test_prefix, 'external_modules_metadata.cfg') @@ -1869,6 +1875,7 @@ def test_external_dependencies(self): deps = ec.dependencies() self.assertEqual(len(deps), 8) + self.assertEqual(ec.dependency_names(), {'intel'}) for idx in [0, 1, 2, 6]: self.assertEqual(deps[idx]['external_module_metadata'], {}) From 097f3942abeca593dd791959b066e21c0c2f8c97 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Oct 2023 11:35:04 +0200 Subject: [PATCH 165/312] Updated tests for syroot template --- test/framework/easyconfig.py | 88 ++++++++++-------------------------- 1 file changed, 23 insertions(+), 65 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index bd02de939d..aed49f291e 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1346,75 +1346,33 @@ def test_start_dir_template(self): self.assertIn('start_dir in extension configure is %s &&' % ext_start_dir, logtxt) self.assertIn('start_dir in extension build is %s &&' % ext_start_dir, logtxt) - def test_sysroot_template_empty(self): - """Test the %(sysroot)s template whenever --sysroot is unset (i.e. None)""" + def test_sysroot_template(self): + """Test the %(sysroot)s template""" - self.contents = textwrap.dedent(""" - name = 'toy' - version = '0.0' - - homepage = 'https://easybuilders.github.io/easybuild' - description = 'Toy C program, 100% toy.' - - toolchain = SYSTEM - - sources = [SOURCE_TAR_GZ] - - preconfigopts = 'echo sysroot in configure is %(sysroot)s && ' - prebuildopts = 'echo sysroot in build is %(sysroot)s && ' - preinstallopts = 'echo sysroot in install is %(sysroot)s && ' - - moduleclass = 'tools' - """) - self.prep() - ec = EasyConfig(self.eb_file) - from easybuild.easyblocks.toy import EB_toy - eb = EB_toy(ec) - # Check behaviour when sysroot is not set (i.e. None) - eb.cfg['sysroot'] = None # Should we define this explicitely? Or rely on this to be the default? - eb.cfg['stop'] = 'extensions' - with self.mocked_stdout_stderr(): - eb.run_all_steps(False) - logtxt = read_file(eb.logfile) - sysroot = "" - self.assertIn('sysroot in configure is %s/ &&' % sysroot, logtxt) - self.assertIn('sysroot in build is %s/ &&' % sysroot, logtxt) - self.assertIn('sysroot in install is %s/ &&' % sysroot, logtxt) - - def test_sysroot_template_non_empty(self): - """Test the %(sysroot)s template whenever --sysroot is unset (i.e. None)""" - - self.contents = textwrap.dedent(""" - name = 'toy' - version = '0.0' - - homepage = 'https://easybuilders.github.io/easybuild' - description = 'Toy C program, 100% toy.' - - toolchain = SYSTEM + test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\nconfigopts = "--some-opt=%(sysroot)s/"' + test_ec_txt += '\nbuildopts = "--some-opt=%(sysroot)s/"' + test_ec_txt += '\ninstallopts = "--some-opt=%(sysroot)s/"' + write_file(test_ec, test_ec_txt) - sources = [SOURCE_TAR_GZ] + # Validate the value of the sysroot template if sysroot is unset (i.e. the build option is None) + ec = EasyConfig(test_ec) + self.assertEqual(ec['configopts'], "--some-opt=/") + self.assertEqual(ec['buildopts'], "--some-opt=/") + self.assertEqual(ec['installopts'], "--some-opt=/") - preconfigopts = 'echo sysroot in configure is %(sysroot)s && ' - prebuildopts = 'echo sysroot in build is %(sysroot)s && ' - preinstallopts = 'echo sysroot in install is %(sysroot)s && ' + # Validate the value of the sysroot template if sysroot is unset (i.e. the build option is None) + # As a test, we'll set the sysroot to self.test_prefix, as it has to be a directory that is guaranteed to exist + update_build_option('sysroot', self.test_prefix) - moduleclass = 'tools' - """) - self.prep() - ec = EasyConfig(self.eb_file) - from easybuild.easyblocks.toy import EB_toy - eb = EB_toy(ec) - # Check behaviour when sysroot is not set (i.e. None) - eb.cfg['sysroot'] = '/tmp' # This should be a path that exists, otherwise EasyBuild complains - eb.cfg['stop'] = 'extensions' - with self.mocked_stdout_stderr(): - eb.run_all_steps(False) - logtxt = read_file(eb.logfile) - sysroot = "" - self.assertIn('sysroot in configure is %s/ &&' % sysroot, logtxt) - self.assertIn('sysroot in build is %s/ &&' % sysroot, logtxt) - self.assertIn('sysroot in install is %s/ &&' % sysroot, logtxt) + ec = EasyConfig(test_ec) + self.assertEqual(ec['configopts'], "--some-opt=%s/" % self.test_prefix) + self.assertEqual(ec['buildopts'], "--some-opt=%s/" % self.test_prefix) + self.assertEqual(ec['installopts'], "--some-opt=%s/" % self.test_prefix) def test_constant_doc(self): """test constant documentation""" From 007667751d99b849c735013b13e35253a022b86c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Oct 2023 11:36:14 +0200 Subject: [PATCH 166/312] Make hound happy --- test/framework/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index aed49f291e..209b92e479 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1351,7 +1351,7 @@ def test_sysroot_template(self): test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') - + test_ec = os.path.join(self.test_prefix, 'test.eb') test_ec_txt = read_file(toy_ec) test_ec_txt += '\nconfigopts = "--some-opt=%(sysroot)s/"' From 0915a995d93dacc5f4478dfcd7d11bdc36331815 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 25 Oct 2023 12:25:16 +0200 Subject: [PATCH 167/312] Deduplicate warnings&errors found in logs and add initial newline in output It isn't useful to report duplicate errors multiple times. And e.g. configure may report a wrong option at the start and end. Hence deduplicate it. Also change the logged warning/error to include a newline & tab in front of each element which makes it nicer to read. E.g.: > build failed (first 300 chars): Found 1 error(s) in command output: > configure: WARNING: unrecognized options: --with-foo, --with-bar (took 2 mins 39 secs) --- easybuild/tools/run.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 8916d80795..19a66db1ca 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -52,7 +52,7 @@ from easybuild.tools.config import ERROR, IGNORE, WARN, build_option from easybuild.tools.hooks import RUN_SHELL_CMD, load_hooks, run_hook from easybuild.tools.py2vs3 import string_type -from easybuild.tools.utilities import trace_msg +from easybuild.tools.utilities import nub, trace_msg _log = fancylogger.getLogger('run', fname=False) @@ -790,7 +790,7 @@ def extract_errors_from_log(log_txt, reg_exps): elif action == WARN: warnings.append(line) break - return warnings, errors + return nub(warnings), nub(errors) def check_log_for_errors(log_txt, reg_exps): @@ -805,8 +805,8 @@ def check_log_for_errors(log_txt, reg_exps): errors_found_in_log += len(warnings) + len(errors) if warnings: - _log.warning("Found %s potential error(s) in command output (output: %s)", + _log.warning("Found %s potential error(s) in command output:\n\t%s", len(warnings), "\n\t".join(warnings)) if errors: - raise EasyBuildError("Found %s error(s) in command output (output: %s)", + raise EasyBuildError("Found %s error(s) in command output:\n\t%s", len(errors), "\n\t".join(errors)) From 8c9b09a994fc60cfd5a4ce9bac0f4bfdf627ff16 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 25 Oct 2023 12:31:34 +0200 Subject: [PATCH 168/312] Explicitely use `set()` instead of `{}` Some Python version may interpret the latter as an (empty) dict instead of a set --- test/framework/easyconfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 5fde75938d..fe921aa52e 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -387,7 +387,7 @@ def test_false_dep_version(self): eb = EasyConfig(self.eb_file) deps = eb.dependencies() self.assertEqual(deps, []) - self.assertEqual(eb.dependency_names(), {}) + self.assertEqual(eb.dependency_names(), set()) def test_extra_options(self): """ extra_options should allow other variables to be stored """ @@ -1655,7 +1655,7 @@ def test_filter_deps(self): init_config(build_options=build_options) ec = EasyConfig(ec_file, validate=False) self.assertEqual(ec.dependencies(), []) - self.assertEqual(ec.dependency_names(), {}) + self.assertEqual(ec.dependency_names(), set()) def test_replaced_easyconfig_parameters(self): """Test handling of replaced easyconfig parameters.""" From 96c7a04bec604ff69608b4c52cceadc1d3734c5e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 25 Oct 2023 13:01:59 +0200 Subject: [PATCH 169/312] Adapt test to new output --- test/framework/run.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/test/framework/run.py b/test/framework/run.py index 1a93ff1a75..d529b41a3a 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -705,8 +705,9 @@ def test_check_log_for_errors(self): "enabling -Werror", "the process crashed with 0" ]) - expected_msg = r"Found 2 error\(s\) in command output "\ - r"\(output: error found\n\tthe process crashed with 0\)" + expected_msg = r"Found 2 error\(s\) in command output:\n"\ + r"\terror found\n"\ + r"\tthe process crashed with 0" # String promoted to list self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text, @@ -718,14 +719,17 @@ def test_check_log_for_errors(self): self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text, [(r"\b(error|crashed)\b", ERROR)]) - expected_msg = "Found 2 potential error(s) in command output " \ - "(output: error found\n\tthe process crashed with 0)" + expected_msg = "Found 2 potential error(s) in command output:\n"\ + "\terror found\n"\ + "\tthe process crashed with 0" init_logging(logfile, silent=True) check_log_for_errors(input_text, [(r"\b(error|crashed)\b", WARN)]) stop_logging(logfile) self.assertIn(expected_msg, read_file(logfile)) - expected_msg = r"Found 2 error\(s\) in command output \(output: error found\n\ttest failed\)" + expected_msg = r"Found 2 error\(s\) in command output:\n"\ + r"\terror found\n"\ + r"\ttest failed" write_file(logfile, '') init_logging(logfile, silent=True) self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text, [ @@ -735,7 +739,7 @@ def test_check_log_for_errors(self): "fail" ]) stop_logging(logfile) - expected_msg = "Found 1 potential error(s) in command output (output: the process crashed with 0)" + expected_msg = "Found 1 potential error(s) in command output:\n\tthe process crashed with 0" self.assertIn(expected_msg, read_file(logfile)) def test_run_cmd_with_hooks(self): From ea000b29e3f6d69ea61964521bc5499596933759 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Oct 2023 14:02:17 +0200 Subject: [PATCH 170/312] Restore original sysroot at the end of test --- test/framework/easyconfig.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 209b92e479..a410e202e4 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1367,6 +1367,7 @@ def test_sysroot_template(self): # Validate the value of the sysroot template if sysroot is unset (i.e. the build option is None) # As a test, we'll set the sysroot to self.test_prefix, as it has to be a directory that is guaranteed to exist + old_sysroot = build_option('sysroot') update_build_option('sysroot', self.test_prefix) ec = EasyConfig(test_ec) @@ -1374,6 +1375,9 @@ def test_sysroot_template(self): self.assertEqual(ec['buildopts'], "--some-opt=%s/" % self.test_prefix) self.assertEqual(ec['installopts'], "--some-opt=%s/" % self.test_prefix) + # Restore original value for sysroot build option + update_build_option('sysroot', old_sysroot) + def test_constant_doc(self): """test constant documentation""" doc = avail_easyconfig_constants() From e6bc40315b48378a7db4c036d837f7033ec75b8f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Oct 2023 14:48:24 +0200 Subject: [PATCH 171/312] Environment reset is not needed at the end of the test, each one starts with clean env. The real problem was that the template_constant_dict has changed, so the expected output is now different - and should include sysroot and its value --- test/framework/easyconfig.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index a410e202e4..5ecf493b2f 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1367,7 +1367,6 @@ def test_sysroot_template(self): # Validate the value of the sysroot template if sysroot is unset (i.e. the build option is None) # As a test, we'll set the sysroot to self.test_prefix, as it has to be a directory that is guaranteed to exist - old_sysroot = build_option('sysroot') update_build_option('sysroot', self.test_prefix) ec = EasyConfig(test_ec) @@ -1375,9 +1374,6 @@ def test_sysroot_template(self): self.assertEqual(ec['buildopts'], "--some-opt=%s/" % self.test_prefix) self.assertEqual(ec['installopts'], "--some-opt=%s/" % self.test_prefix) - # Restore original value for sysroot build option - update_build_option('sysroot', old_sysroot) - def test_constant_doc(self): """test constant documentation""" doc = avail_easyconfig_constants() @@ -3266,6 +3262,7 @@ def test_template_constant_dict(self): 'nameletter': 'g', 'nameletterlower': 'g', 'parallel': None, + 'sysroot': '', 'toolchain_name': 'foss', 'toolchain_version': '2018a', 'version': '1.5', From f2bae7f811ae6c12bbdae922caf0f68c00858db8 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Oct 2023 15:09:20 +0200 Subject: [PATCH 172/312] One more change in expected output. Probably need one more after this, lets try... --- test/framework/easyconfig.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 5ecf493b2f..742bfe50f9 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3344,6 +3344,7 @@ def test_template_constant_dict(self): 'pyminver': '7', 'pyshortver': '3.7', 'pyver': '3.7.2', + 'sysroot': '', 'version': '0.01', 'version_major': '0', 'version_major_minor': '0.01', From a4105265c1632f0cef69f26b5c8df464bae5ed38 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Oct 2023 15:59:28 +0200 Subject: [PATCH 173/312] Fix hopefully last reference in test_template_constant_dict to account for addition of sysroot template --- test/framework/easyconfig.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 742bfe50f9..ddc0f78b57 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3409,6 +3409,7 @@ def test_template_constant_dict(self): 'namelower': 'foo', 'nameletter': 'f', 'nameletterlower': 'f', + 'sysroot': '', 'version': '1.2.3', 'version_major': '1', 'version_major_minor': '1.2', From 9c4c794bbebb5d66099b1ace51177985e37c00c6 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 26 Oct 2023 16:18:29 +0200 Subject: [PATCH 174/312] fix findPythonDeps when called with an absolute path to an EC `eb --missing` returns a list with filenames. So an absolute path passed to `--ec` isn't matched which leads to an error stating you should install the EasyConfig that you are trying to find the dependencies of. Fix by getting the filename/basename first. --- easybuild/scripts/findPythonDeps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index 663926a9d6..ef284a7e21 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -171,8 +171,9 @@ def print_deps(package, verbose): capture_stderr=False, action_desc='Get missing dependencies' ) + excluded_dep = '(%s)' % os.path.basename(args.ec) missing_deps = [dep for dep in missing_dep_out.split('\n') - if dep.startswith('*') and '(%s)' % args.ec not in dep + if dep.startswith('*') and excluded_dep not in dep ] if missing_deps: print('You need to install all modules on which %s depends first!' % args.ec) From dfe010283b132690fb2051180dfb8ecfe272b267 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 27 Oct 2023 09:14:59 +0200 Subject: [PATCH 175/312] Show stderr output on failing command even when not requested to be captured This helps diagnosing failures for e.g. `eb --missing` (called with `capture_stderr=False`) but which may print errors (e.g. failed parsing of some ECs) to stderr --- easybuild/scripts/findPythonDeps.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index ef284a7e21..4f4200de09 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -55,9 +55,11 @@ def run_cmd(arguments, action_desc, capture_stderr=True, **kwargs): extra_args['universal_newlines'] = True stderr = subprocess.STDOUT if capture_stderr else subprocess.PIPE p = subprocess.Popen(arguments, stdout=subprocess.PIPE, stderr=stderr, **extra_args) - out, _ = p.communicate() + out, err = p.communicate() if p.returncode != 0: - raise RuntimeError('Failed to %s: %s' % (action_desc, out)) + if err: + err = "\nSTDERR:\n" + err + raise RuntimeError('Failed to %s: %s%s' % (action_desc, out, err)) return out From cf0bd4f2e68c2b5521dfb894c4e96dc820a179b9 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 27 Oct 2023 10:17:02 +0200 Subject: [PATCH 176/312] fix findPythonDeps when using a relative path as the `--ec` argument --- easybuild/scripts/findPythonDeps.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index 4f4200de09..d6e496a048 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -182,12 +182,14 @@ def print_deps(package, verbose): print('\n\t'.join(['Missing:'] + missing_deps)) sys.exit(1) + # If the --ec argument is a (relative) existing path make it absolute so we can find it after the chdir + ec_arg = os.path.abspath(args.ec) if os.path.exists(args.ec) else args.ec with temporary_directory() as tmp_dir: old_dir = os.getcwd() os.chdir(tmp_dir) if args.verbose: print('Running EasyBuild to get build environment') - run_cmd(['eb', args.ec, '--dump-env', '--force'], action_desc='Dump build environment') + run_cmd(['eb', ec_arg, '--dump-env', '--force'], action_desc='Dump build environment') os.chdir(old_dir) cmd = "source %s/*.env && python %s '%s'" % (tmp_dir, sys.argv[0], args.package) From a842a38a7cfd44119e31ef4144efdaac1dd4cd15 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 27 Oct 2023 10:28:30 +0200 Subject: [PATCH 177/312] fix broken test for reasons_for_closing, which fails because commit status of easyconfigs PR is no longer available --- test/framework/github.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/framework/github.py b/test/framework/github.py index c9bfbaedd3..f011c89b4f 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -345,7 +345,6 @@ def test_github_reasons_for_closing(self): self.assertIsInstance(res, list) self.assertEqual(stderr.strip(), "WARNING: Using easyconfigs from closed PR #16080") patterns = [ - "Status of last commit is SUCCESS", "Last comment on", "No activity since", "* c-ares-1.18.1", From 642fe35a84a0b6c47a10b3c0d1ef48b4ff08701f Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Sat, 28 Oct 2023 16:26:58 +0800 Subject: [PATCH 178/312] prepare release notes for EasyBuild v4.8.2 + bump version to 4.8.2 --- RELEASE_NOTES | 22 ++++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 5bd07d4306..8365871584 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,28 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.8.2 (29 October 2023) +------------------------ + +update/bugfix release + +- various enhancements, including: + - add support for `%(sysroot)s` template value (#4359) + - add `dependency_names` method to `EasyConfig` class to get set of names of (direct) dependencies (#4360) +- various bug fixes, including: + - add CI workflow to run unit tests with Python 2 (again) (#4333) + - fix typo in help message for --silence-hook-trigger (#4343) + - include major version (`*majver`) templates in auto-generated documentation (#4347) + - reset `tempfile.tempdir` to `None` to avoid that tmpdir path gets progressively deeper with each easystack item (#4350) + - fix parallellisation typo (#4352) + - minor help fix (#4355) + - fix `findPythonDeps.py` script when called with an absolute path to an easyconfig (#4365) + - fix broken test for `reasons_for_closing`, which fails because commit status of easyconfigs PR is no longer available (#4366) +- other changes: + - reduce number of CI jobs by testing for Lua and Tcl module syntax in a single CI job (#4192) + - clean up stray regex.search in `test_checksum_step` (#4345) + + v4.8.1 (11 September 2023) -------------------------- diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index e34584df36..8582267629 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -45,7 +45,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.8.2.dev0') +VERSION = LooseVersion('4.8.2') UNKNOWN = 'UNKNOWN' From 49deccdceddec80b6b782a6b99af19c3d98c78ea Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 28 Oct 2023 16:48:38 +0200 Subject: [PATCH 179/312] tweak release notes for EasyBuild v4.8.2 --- RELEASE_NOTES | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 8365871584..a43ce28535 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -14,16 +14,13 @@ update/bugfix release - add `dependency_names` method to `EasyConfig` class to get set of names of (direct) dependencies (#4360) - various bug fixes, including: - add CI workflow to run unit tests with Python 2 (again) (#4333) - - fix typo in help message for --silence-hook-trigger (#4343) + - fix typo in help message for `--silence-hook-trigger` (#4343) - include major version (`*majver`) templates in auto-generated documentation (#4347) - reset `tempfile.tempdir` to `None` to avoid that tmpdir path gets progressively deeper with each easystack item (#4350) - - fix parallellisation typo (#4352) - - minor help fix (#4355) - - fix `findPythonDeps.py` script when called with an absolute path to an easyconfig (#4365) + - fix `findPythonDeps.py` script when called with an (absolute or relative) path to an easyconfig instead of a filename (#4365) - fix broken test for `reasons_for_closing`, which fails because commit status of easyconfigs PR is no longer available (#4366) - other changes: - reduce number of CI jobs by testing for Lua and Tcl module syntax in a single CI job (#4192) - - clean up stray regex.search in `test_checksum_step` (#4345) v4.8.1 (11 September 2023) From f4dda1ea638f11f16652ab00ca88b72b96ea7762 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Sun, 29 Oct 2023 11:16:30 +0100 Subject: [PATCH 180/312] Fix test_load test for EnvironmentModules 4.2+ test_load expects that version 4.2+ of Environment Modules reload an already loaded module. As of version 5.3, already loaded modules are not reloaded unless one of their dependent module is unloaded or loaded. --- test/framework/modules.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 7bf87130bd..738cc79595 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -437,9 +437,8 @@ def test_load(self): # if GCC is loaded again, $EBROOTGCC should be set again, and GCC should be listed last self.modtool.load(['GCC/6.4.0-2.28']) - # environment modules v4.0 does not reload already loaded modules, will be changed in v4.2 - modtool_ver = StrictVersion(self.modtool.version) - if not isinstance(self.modtool, EnvironmentModules) or modtool_ver >= StrictVersion('4.2'): + # environment modules v4+ does not reload already loaded modules + if not isinstance(self.modtool, EnvironmentModules): self.assertTrue(os.environ.get('EBROOTGCC')) if isinstance(self.modtool, Lmod): From 94ff50f8df8c0dfa05c6bec94bf32900112b8100 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Sun, 29 Oct 2023 11:26:15 +0100 Subject: [PATCH 181/312] Ensure correct configuration for EnvironmentModules Defines environment variables when initializing EnvironmentModules object to ensure that this module tool will not be influenced by external configuration. Setup Environment Modules configuration to ensure module search behaves like EasyBuild expects (match module name start, case sensitive, return in-depth modulepath content, ignore cache file). Also defines a basic output configuration not to get unexpected content like tags or variants. This change helps to pass "test_avail" test with Environment Modules v5.0+. --- easybuild/tools/modules.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 386643e706..aa739f7563 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1328,6 +1328,29 @@ class EnvironmentModules(EnvironmentModulesTcl): MAX_VERSION = None VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d\S*)\s' + def __init__(self, *args, **kwargs): + """Constructor, set Environment Modules-specific class variable values.""" + # ensure in-depth modulepath search (MODULES_AVAIL_INDEPTH has been introduced in v4.3) + setvar('MODULES_AVAIL_INDEPTH', '1', verbose=False) + # match against module name start (MODULES_SEARCH_MATCH has been introduced in v4.3) + setvar('MODULES_SEARCH_MATCH', 'starts_with', verbose=False) + # ensure no debug message (MODULES_VERBOSITY has been introduced in v4.3) + setvar('MODULES_VERBOSITY', 'normal', verbose=False) + # make module search case sensitive (search is case insensitive by default since v5.0) + setvar('MODULES_ICASE', 'never', verbose=False) + # disable extended default (introduced in v4.4 and enabled by default in v5.0) + setvar('MODULES_EXTENDED_DEFAULT', '0', verbose=False) + # hard disable output redirection, output messages are expected on stderr + setvar('MODULES_REDIRECT_OUTPUT', '0', verbose=False) + # make sure modulefile cache is ignored (cache mechanism supported since v5.3) + setvar('MODULES_IGNORE_CACHE', '1', verbose=False) + # ensure only module names are returned on avail (MODULES_AVAIL_TERSE_OUTPUT added in v4.7) + setvar('MODULES_AVAIL_TERSE_OUTPUT', '', verbose=False) + # ensure only module names are returned on list (MODULES_LIST_TERSE_OUTPUT added in v4.7) + setvar('MODULES_LIST_TERSE_OUTPUT', '', verbose=False) + + super(EnvironmentModules, self).__init__(*args, **kwargs) + def check_module_output(self, cmd, stdout, stderr): """Check output of 'module' command, see if if is potentially invalid.""" if "_mlstatus = False" in stdout: From f297d321b276730a6a5b806e60e266342012b999 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Sun, 29 Oct 2023 11:35:32 +0100 Subject: [PATCH 182/312] Setenv value may be enclosed in curly braces on EnvironmentModules Starting Environment Modules 4.2, value of setenv statement in "module show" output is enclosed in curly braces if it contains spaces. This change helps to pass "test_get_setenv_value_from_modulefile" test with Environment Modules v4.2+ --- easybuild/tools/modules.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index aa739f7563..e0eda91aeb 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1358,6 +1358,27 @@ def check_module_output(self, cmd, stdout, stderr): else: self.log.debug("No errors detected when running module command '%s'", cmd) + def get_setenv_value_from_modulefile(self, mod_name, var_name): + """ + Get value for specific 'setenv' statement from module file for the specified module. + + :param mod_name: module name + :param var_name: name of the variable being set for which value should be returned + """ + # Tcl-based module tools produce "module show" output with setenv statements like: + # "setenv GCC_PATH /opt/gcc/8.3.0" + # "setenv VAR {some text} + # - line starts with 'setenv' + # - whitespace (spaces & tabs) around variable name + # - curly braces around value if it contain spaces + value = super(EnvironmentModules, self).get_setenv_value_from_modulefile(mod_name=mod_name, + var_name=var_name) + + if value: + value = value.strip('{}') + + return value + class Lmod(ModulesTool): """Interface to Lmod.""" From e9f190c7802d340f09b177743f77e1a022a964f7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 4 Nov 2023 18:05:16 +0100 Subject: [PATCH 183/312] only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9 --- .github/workflows/unit_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 5b09fc654b..ec98f9df03 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -103,9 +103,9 @@ jobs: # and are only run after the PR gets merged GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}} run: | - # don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools, to avoid hitting GitHub rate limit; + # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit; # tests that require a GitHub token are skipped automatically when no GitHub token is available - if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]]; then + if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.[69] ]]; then if [ ! -z $GITHUB_TOKEN ]; then SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"; python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; From 13da6438282a5cc052724feb230e62285687b851 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 4 Nov 2023 21:07:02 +0100 Subject: [PATCH 184/312] bump version to 4.9.0dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 8582267629..655f7d0a49 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -45,7 +45,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.8.2') +VERSION = LooseVersion('4.9.0.dev0') UNKNOWN = 'UNKNOWN' From 316a89beb828a3b61d4ccae80a7b2b40cc68b7ee Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 6 Nov 2023 09:35:54 +0100 Subject: [PATCH 185/312] fix mocked RunShellCmdResult in systemtools tests --- test/framework/systemtools.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 83aec64336..7c7d7698b3 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -340,7 +340,8 @@ def mocked_run_shell_cmd(cmd, **kwargs): "ulimit -u": '40', } if cmd in known_cmds: - return RunShellCmdResult(cmd=cmd, exit_code=0, output=known_cmds[cmd], stderr=None, work_dir=os.getcwd()) + return RunShellCmdResult(cmd=cmd, exit_code=0, output=known_cmds[cmd], stderr=None, work_dir=os.getcwd(), + out_file=None, err_file=None) else: return run_shell_cmd(cmd, **kwargs) @@ -775,7 +776,8 @@ def test_gcc_version_darwin(self): st.get_os_type = lambda: st.DARWIN out = "Apple LLVM version 7.0.0 (clang-700.1.76)" cwd = os.getcwd() - mocked_run_res = RunShellCmdResult(cmd="gcc --version", exit_code=0, output=out, stderr=None, work_dir=cwd) + mocked_run_res = RunShellCmdResult(cmd="gcc --version", exit_code=0, output=out, stderr=None, work_dir=cwd, + out_file=None, err_file=None) st.run_shell_cmd = lambda *args, **kwargs: mocked_run_res self.assertEqual(get_gcc_version(), None) From ca9186ecf0a47d36fb510ca09e735fb5d652ddd9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 6 Nov 2023 10:13:00 +0100 Subject: [PATCH 186/312] only log shell command output when output_file option is enabled in run_shell_cmd --- easybuild/tools/run.py | 50 +++++++++++++++++++++++------------------- test/framework/run.py | 6 ++--- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index c8ec8373c7..add1a1f7cd 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -107,13 +107,13 @@ def pad_4_spaces(msg): pad_4_spaces(f"working directory -> {self.cmd_result.work_dir}"), ] - if self.cmd_result.stderr is None: + if self.cmd_result.stderr is None and self.cmd_result.out_file is not None: error_info.append(pad_4_spaces(f"output (stdout + stderr) -> {self.cmd_result.out_file}")) else: - error_info.extend([ - pad_4_spaces(f"output (stdout) -> {self.cmd_result.out_file}"), - pad_4_spaces(f"error/warnings (stderr) -> {self.cmd_result.err_file}"), - ]) + if self.cmd_result.out_file is not None: + error_info.append(pad_4_spaces(f"output (stdout) -> {self.cmd_result.out_file}")) + if self.cmd_result.err_file is not None: + error_info.append(pad_4_spaces(f"error/warnings (stderr) -> {self.cmd_result.err_file}")) caller_file_name, caller_line_nr, caller_function_name = self.caller_info called_from_info = f"'{caller_function_name}' function in {caller_file_name} (line {caller_line_nr})" @@ -174,7 +174,7 @@ def cache_aware_func(cmd, *args, **kwargs): @run_shell_cmd_cache def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, shell=True, - output_file=False, stream_output=False, asynchronous=False, with_hooks=True, + output_file=True, stream_output=False, asynchronous=False, with_hooks=True, qa_patterns=None, qa_wait_patterns=None): """ Run specified (interactive) shell command, and capture output + exit code. @@ -224,17 +224,22 @@ def to_cmd_str(cmd): work_dir = os.getcwd() cmd_str = to_cmd_str(cmd) - cmd_name = cmd_str.split(' ')[0] + cmd_name = os.path.basename(cmd_str.split(' ')[0]) # temporary output file(s) for command output - tmpdir = tempfile.mkdtemp(prefix='run-shell-cmd-') - cmd_out_fp = os.path.join(tmpdir, f'{cmd_name}.out') - _log.info(f'run_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}') - if split_stderr: - cmd_err_fp = os.path.join(tmpdir, f'{cmd_name}.err') - _log.info(f'run_cmd: Errors and warnings of "{cmd_str}" will be logged to {cmd_err_fp}') + if output_file: + toptmpdir = os.path.join(tempfile.gettempdir(), 'run-shell-cmd-output') + os.makedirs(toptmpdir, exist_ok=True) + tmpdir = tempfile.mkdtemp(dir=toptmpdir, prefix=f'{cmd_name}-') + cmd_out_fp = os.path.join(tmpdir, f'out.txt') + _log.info(f'run_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}') + if split_stderr: + cmd_err_fp = os.path.join(tmpdir, f'err.txt') + _log.info(f'run_cmd: Errors and warnings of "{cmd_str}" will be logged to {cmd_err_fp}') + else: + cmd_err_fp = None else: - cmd_err_fp = None + cmd_out_fp, cmd_err_fp = None, None # early exit in 'dry run' mode, after printing the command that would be run (unless 'hidden' is enabled) if not in_dry_run and build_option('extended_dry_run'): @@ -279,14 +284,15 @@ def to_cmd_str(cmd): stderr = proc.stderr.decode('utf-8', 'ignore') if split_stderr else None # store command output to temporary file(s) - try: - with open(cmd_out_fp, 'w') as fp: - fp.write(output) - if split_stderr: - with open(cmd_err_fp, 'w') as fp: - fp.write(stderr) - except IOError as err: - raise EasyBuildError(f"Failed to dump command output to temporary file: {err}") + if output_file: + try: + with open(cmd_out_fp, 'w') as fp: + fp.write(output) + if split_stderr: + with open(cmd_err_fp, 'w') as fp: + fp.write(stderr) + except IOError as err: + raise EasyBuildError(f"Failed to dump command output to temporary file: {err}") res = RunShellCmdResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr, work_dir=work_dir, out_file=cmd_out_fp, err_file=cmd_err_fp) diff --git a/test/framework/run.py b/test/framework/run.py index 91b03d1b44..1b42e961ec 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -356,7 +356,7 @@ def handler(signum, _): r"^\s+exit code\s* -> -9", r"^\s+working directory\s* -> " + work_dir, r"^\s+called from\s* -> 'test_run_shell_cmd_fail' function in .*/test/.*/run.py \(line [0-9]+\)", - r"^\s+output \(stdout \+ stderr\)\s* -> .*/run-shell-cmd-.*/kill.out", + r"^\s+output \(stdout \+ stderr\)\s* -> .*/run-shell-cmd-output/kill-.*/out.txt", ] for pattern in patterns: regex = re.compile(pattern, re.M) @@ -391,8 +391,8 @@ def handler(signum, _): r"^\s+exit code\s+ -> -9", r"^\s+working directory\s+ -> " + work_dir, r"^\s+called from\s+ -> 'test_run_shell_cmd_fail' function in .*/test/.*/run.py \(line [0-9]+\)", - r"^\s+output \(stdout\)\s+ -> .*/run-shell-cmd-.*/kill.out", - r"^\s+error/warnings \(stderr\)\s+ -> .*/run-shell-cmd-.*/kill.err", + r"^\s+output \(stdout\)\s+ -> .*/run-shell-cmd-output/kill-.*/out.txt", + r"^\s+error/warnings \(stderr\)\s+ -> .*/run-shell-cmd-output/kill-.*/err.txt", ] for pattern in patterns: regex = re.compile(pattern, re.M) From 95bd45efbe54966cd50b19d36ee01c2ce21490a9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 6 Nov 2023 10:13:59 +0100 Subject: [PATCH 187/312] disable storing output of shell commands to temporary file in tools/modules.py and tools/systemtools.py --- easybuild/tools/modules.py | 4 ++-- easybuild/tools/systemtools.py | 39 +++++++++++++++++++--------------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index e9ede9c80e..60f2929a83 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -312,7 +312,7 @@ def check_module_function(self, allow_mismatch=False, regex=None): output, exit_code = None, 1 else: cmd = "type module" - res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=False, hidden=True) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=False, hidden=True, output_file=False) output, exit_code = res.output, res.exit_code if regex is None: @@ -821,7 +821,7 @@ def run_module(self, *args, **kwargs): cmd = ' '.join(cmd_list) # note: module commands are always run in dry mode, and are kept hidden in trace and dry run output res = run_shell_cmd(cmd_list, env=environ, fail_on_error=False, shell=False, split_stderr=True, - hidden=True, in_dry_run=True) + hidden=True, in_dry_run=True, output_file=False) # stdout will contain python code (to change environment etc) # stderr will contain text (just like the normal module command) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index ef4cccf05d..32caf0b4a0 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -273,7 +273,7 @@ def get_avail_core_count(): core_cnt = int(sum(sched_getaffinity())) else: # BSD-type systems - res = run_shell_cmd('sysctl -n hw.ncpu', in_dry_run=True, hidden=True, with_hooks=False) + res = run_shell_cmd('sysctl -n hw.ncpu', in_dry_run=True, hidden=True, with_hooks=False, output_file=False) try: if int(res.output) > 0: core_cnt = int(res.output) @@ -310,7 +310,7 @@ def get_total_memory(): elif os_type == DARWIN: cmd = "sysctl -n hw.memsize" _log.debug("Trying to determine total memory size on Darwin via cmd '%s'", cmd) - res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False) + res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False) if res.exit_code == 0: memtotal = int(res.output.strip()) // (1024**2) @@ -392,14 +392,15 @@ def get_cpu_vendor(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.vendor" - res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, output_file=False) out = res.output.strip() if res.exit_code == 0 and out in VENDOR_IDS: vendor = VENDOR_IDS[out] _log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd)) else: cmd = "sysctl -n machdep.cpu.brand_string" - res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, + output_file=False) out = res.output.strip().split(' ')[0] if res.exit_code == 0 and out in CPU_VENDORS: vendor = out @@ -502,7 +503,7 @@ def get_cpu_model(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.brand_string" - res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False) + res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False) if res.exit_code == 0: model = res.output.strip() _log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model)) @@ -547,7 +548,7 @@ def get_cpu_speed(): elif os_type == DARWIN: cmd = "sysctl -n hw.cpufrequency_max" _log.debug("Trying to determine CPU frequency on Darwin via cmd '%s'" % cmd) - res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False) + res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False) out = res.output.strip() cpu_freq = None if res.exit_code == 0 and out: @@ -595,7 +596,8 @@ def get_cpu_features(): for feature_set in ['extfeatures', 'features', 'leaf7_features']: cmd = "sysctl -n machdep.cpu.%s" % feature_set _log.debug("Trying to determine CPU features on Darwin via cmd '%s'", cmd) - res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, fail_on_error=False, with_hooks=False) + res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, fail_on_error=False, with_hooks=False, + output_file=False) if res.exit_code == 0: cpu_feat.extend(res.output.strip().lower().split()) @@ -622,7 +624,8 @@ def get_gpu_info(): try: cmd = "nvidia-smi --query-gpu=gpu_name,driver_version --format=csv,noheader" _log.debug("Trying to determine NVIDIA GPU info on Linux via cmd '%s'", cmd) - res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, + output_file=False) if res.exit_code == 0: for line in res.output.strip().split('\n'): nvidia_gpu_info = gpu_info.setdefault('NVIDIA', {}) @@ -640,13 +643,15 @@ def get_gpu_info(): try: cmd = "rocm-smi --showdriverversion --csv" _log.debug("Trying to determine AMD GPU driver on Linux via cmd '%s'", cmd) - res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, + output_file=False) if res.exit_code == 0: amd_driver = res.output.strip().split('\n')[1].split(',')[1] cmd = "rocm-smi --showproductname --csv" _log.debug("Trying to determine AMD GPU info on Linux via cmd '%s'", cmd) - res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, + output_file=False) if res.exit_code == 0: for line in res.output.strip().split('\n')[1:]: amd_card_series = line.split(',')[1] @@ -865,7 +870,7 @@ def check_os_dependency(dep): pkg_cmd_flag.get(pkg_cmd), dep, ]) - res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, output_file=False) found = res.exit_code == 0 if found: break @@ -877,7 +882,7 @@ def check_os_dependency(dep): # try locate if it's available if not found and which('locate'): cmd = 'locate -c --regexp "/%s$"' % dep - res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, output_file=False) try: found = (res.exit_code == 0 and int(res.output.strip()) > 0) except ValueError: @@ -893,7 +898,7 @@ def get_tool_version(tool, version_option='--version', ignore_ec=False): Output is returned as a single-line string (newlines are replaced by '; '). """ res = run_shell_cmd(' '.join([tool, version_option]), fail_on_error=False, in_dry_run=True, - hidden=True, with_hooks=False) + hidden=True, with_hooks=False, output_file=False) if not ignore_ec and res.exit_code: _log.warning("Failed to determine version of %s using '%s %s': %s" % (tool, tool, version_option, res.output)) return UNKNOWN @@ -905,7 +910,7 @@ def get_gcc_version(): """ Process `gcc --version` and return the GCC version. """ - res = run_shell_cmd('gcc --version', fail_on_error=False, in_dry_run=True, hidden=True) + res = run_shell_cmd('gcc --version', fail_on_error=False, in_dry_run=True, hidden=True, output_file=False) gcc_ver = None if res.exit_code: _log.warning("Failed to determine the version of GCC: %s", res.output) @@ -961,7 +966,7 @@ def get_linked_libs_raw(path): or None for other types of files. """ - res = run_shell_cmd("file %s" % path, fail_on_error=False, hidden=True) + res = run_shell_cmd("file %s" % path, fail_on_error=False, hidden=True, output_file=False) if res.exit_code: fail_msg = "Failed to run 'file %s': %s" % (path, res.output) _log.warning(fail_msg) @@ -996,7 +1001,7 @@ def get_linked_libs_raw(path): # take into account that 'ldd' may fail for strange reasons, # like printing 'not a dynamic executable' when not enough memory is available # (see also https://bugzilla.redhat.com/show_bug.cgi?id=1817111) - res = run_shell_cmd(linked_libs_cmd, fail_on_error=False, hidden=True) + res = run_shell_cmd(linked_libs_cmd, fail_on_error=False, hidden=True, output_file=False) if res.exit_code == 0: linked_libs_out = res.output else: @@ -1178,7 +1183,7 @@ def get_default_parallelism(): # No cache -> Calculate value from current system values par = get_avail_core_count() # determine max user processes via ulimit -u - res = run_shell_cmd("ulimit -u", in_dry_run=True, hidden=True) + res = run_shell_cmd("ulimit -u", in_dry_run=True, hidden=True, output_file=False) try: if res.output.startswith("unlimited"): maxuserproc = 2 ** 32 - 1 From 121e199c1d4b24eab608275e1e0004f435da85b6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 6 Nov 2023 10:15:49 +0100 Subject: [PATCH 188/312] don't use f-string for path to output files since there are no placeholders in run_shell_cmd --- easybuild/tools/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index add1a1f7cd..70b75c0018 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -231,10 +231,10 @@ def to_cmd_str(cmd): toptmpdir = os.path.join(tempfile.gettempdir(), 'run-shell-cmd-output') os.makedirs(toptmpdir, exist_ok=True) tmpdir = tempfile.mkdtemp(dir=toptmpdir, prefix=f'{cmd_name}-') - cmd_out_fp = os.path.join(tmpdir, f'out.txt') + cmd_out_fp = os.path.join(tmpdir, 'out.txt') _log.info(f'run_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}') if split_stderr: - cmd_err_fp = os.path.join(tmpdir, f'err.txt') + cmd_err_fp = os.path.join(tmpdir, 'err.txt') _log.info(f'run_cmd: Errors and warnings of "{cmd_str}" will be logged to {cmd_err_fp}') else: cmd_err_fp = None From 98a57205c6b9a72d5314196722b08c40cea53408 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 6 Nov 2023 20:33:56 +0100 Subject: [PATCH 189/312] take into account that path to file with command output may be None in cmd_trace_msg --- easybuild/tools/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 70b75c0018..c60d384c84 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -348,7 +348,8 @@ def cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp): ] if stdin: lines.append(f"\t[input: {stdin}]") - lines.append(f"\t[output saved to {cmd_out_fp}]") + if cmd_out_fp: + lines.append(f"\t[output saved to {cmd_out_fp}]") if cmd_err_fp: lines.append(f"\t[errors/warnings saved to {cmd_err_fp}]") From f46dd91b1a4dd216779c9f9c3787125b2ec8b43b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 6 Nov 2023 20:40:02 +0100 Subject: [PATCH 190/312] unpack RunShellCmdResult instance in RunShellCmdError constructor into individual fields --- easybuild/tools/run.py | 29 ++++++++++++++++++----------- test/framework/run.py | 20 ++++++++++---------- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index c60d384c84..6deccc2703 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -84,8 +84,15 @@ class RunShellCmdError(BaseException): def __init__(self, cmd_result, caller_info, *args, **kwargs): """Constructor for RunShellCmdError.""" - self.cmd_result = cmd_result - self.cmd_name = cmd_result.cmd.split(' ')[0] + self.cmd = cmd_result.cmd + self.cmd_name = os.path.basename(self.cmd.split(' ')[0]) + self.exit_code = cmd_result.exit_code + self.work_dir = cmd_result.work_dir + self.output = cmd_result.output + self.out_file = cmd_result.out_file + self.stderr = cmd_result.stderr + self.err_file = cmd_result.err_file + self.caller_info = caller_info msg = f"Shell command '{self.cmd_name}' failed!" @@ -102,18 +109,18 @@ def pad_4_spaces(msg): error_info = [ '', "ERROR: Shell command failed!", - pad_4_spaces(f"full command -> {self.cmd_result.cmd}"), - pad_4_spaces(f"exit code -> {self.cmd_result.exit_code}"), - pad_4_spaces(f"working directory -> {self.cmd_result.work_dir}"), + pad_4_spaces(f"full command -> {self.cmd}"), + pad_4_spaces(f"exit code -> {self.exit_code}"), + pad_4_spaces(f"working directory -> {self.work_dir}"), ] - if self.cmd_result.stderr is None and self.cmd_result.out_file is not None: - error_info.append(pad_4_spaces(f"output (stdout + stderr) -> {self.cmd_result.out_file}")) + if self.stderr is None and self.out_file is not None: + error_info.append(pad_4_spaces(f"output (stdout + stderr) -> {self.out_file}")) else: - if self.cmd_result.out_file is not None: - error_info.append(pad_4_spaces(f"output (stdout) -> {self.cmd_result.out_file}")) - if self.cmd_result.err_file is not None: - error_info.append(pad_4_spaces(f"error/warnings (stderr) -> {self.cmd_result.err_file}")) + if self.out_file is not None: + error_info.append(pad_4_spaces(f"output (stdout) -> {self.out_file}")) + if self.err_file is not None: + error_info.append(pad_4_spaces(f"error/warnings (stderr) -> {self.err_file}")) caller_file_name, caller_line_nr, caller_function_name = self.caller_info called_from_info = f"'{caller_function_name}' function in {caller_file_name} (line {caller_line_nr})" diff --git a/test/framework/run.py b/test/framework/run.py index 1b42e961ec..948eac51b1 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -333,12 +333,12 @@ def handler(signum, _): self.assertFalse("This should never be reached, RunShellCmdError should occur!") except RunShellCmdError as err: self.assertEqual(str(err), "Shell command 'kill' failed!") - self.assertEqual(err.cmd_result.cmd, "kill -9 $$") + self.assertEqual(err.cmd, "kill -9 $$") self.assertEqual(err.cmd_name, 'kill') - self.assertEqual(err.cmd_result.exit_code, -9) - self.assertEqual(err.cmd_result.work_dir, work_dir) - self.assertEqual(err.cmd_result.output, '') - self.assertEqual(err.cmd_result.stderr, None) + self.assertEqual(err.exit_code, -9) + self.assertEqual(err.work_dir, work_dir) + self.assertEqual(err.output, '') + self.assertEqual(err.stderr, None) self.assertTrue(isinstance(err.caller_info, tuple)) self.assertEqual(len(err.caller_info), 3) self.assertEqual(err.caller_info[0], __file__) @@ -368,12 +368,12 @@ def handler(signum, _): self.assertFalse("This should never be reached, RunShellCmdError should occur!") except RunShellCmdError as err: self.assertEqual(str(err), "Shell command 'kill' failed!") - self.assertEqual(err.cmd_result.cmd, "kill -9 $$") + self.assertEqual(err.cmd, "kill -9 $$") self.assertEqual(err.cmd_name, 'kill') - self.assertEqual(err.cmd_result.exit_code, -9) - self.assertEqual(err.cmd_result.work_dir, work_dir) - self.assertEqual(err.cmd_result.output, '') - self.assertEqual(err.cmd_result.stderr, '') + self.assertEqual(err.exit_code, -9) + self.assertEqual(err.work_dir, work_dir) + self.assertEqual(err.output, '') + self.assertEqual(err.stderr, '') self.assertTrue(isinstance(err.caller_info, tuple)) self.assertEqual(len(err.caller_info), 3) self.assertEqual(err.caller_info[0], __file__) From 8620e64b9cbfa43f1a55f9c0b22a2cd78d44b166 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Nov 2023 13:28:39 +0100 Subject: [PATCH 191/312] simplify logic for printing path to stdout/stderr output files in RunShellCmdError.print --- easybuild/tools/run.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 6deccc2703..589528a6ab 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -114,13 +114,13 @@ def pad_4_spaces(msg): pad_4_spaces(f"working directory -> {self.work_dir}"), ] - if self.stderr is None and self.out_file is not None: - error_info.append(pad_4_spaces(f"output (stdout + stderr) -> {self.out_file}")) - else: - if self.out_file is not None: - error_info.append(pad_4_spaces(f"output (stdout) -> {self.out_file}")) - if self.err_file is not None: - error_info.append(pad_4_spaces(f"error/warnings (stderr) -> {self.err_file}")) + if self.out_file is not None: + # if there's no separate file for error/warnings, then out_file includes both stdout + stderr + out_info_msg = "output (stdout + stderr)" if self.err_file is None else "output (stdout) " + error_info.append(pad_4_spaces(f"{out_info_msg} -> {self.out_file}")) + + if self.err_file is not None: + error_info.append(pad_4_spaces(f"error/warnings (stderr) -> {self.err_file}")) caller_file_name, caller_line_nr, caller_function_name = self.caller_info called_from_info = f"'{caller_function_name}' function in {caller_file_name} (line {caller_line_nr})" From ee99d87e910a210638eb13e841c2920f152459c0 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 8 Nov 2023 16:51:16 +0000 Subject: [PATCH 192/312] Use -qopenmp instead of -fiopenmp for OpenMP in OneAPI Recommended in porting guide: qopenmp, unlike fiopenmp, works for both classic and oneapi compilers https://www.intel.com/content/www/us/en/developer/articles/guide/porting-guide-for-ifort-to-ifx.html Fixes #4376 --- easybuild/toolchains/compiler/intel_compilers.py | 5 +++-- test/framework/toolchain.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/toolchains/compiler/intel_compilers.py b/easybuild/toolchains/compiler/intel_compilers.py index ef537d9315..ae97dfa87d 100644 --- a/easybuild/toolchains/compiler/intel_compilers.py +++ b/easybuild/toolchains/compiler/intel_compilers.py @@ -109,8 +109,9 @@ def set_variables(self): self.options.options_map['loose'] = ['fp-model fast'] # fp-model fast=2 gives "warning: overriding '-ffp-model=fast=2' option with '-ffp-model=fast'" self.options.options_map['veryloose'] = ['fp-model fast'] - # recommended in porting guide - self.options.options_map['openmp'] = ['fiopenmp'] + # recommended in porting guide: qopenmp, unlike fiopenmp, works for both classic and oneapi compilers + # https://www.intel.com/content/www/us/en/developer/articles/guide/porting-guide-for-ifort-to-ifx.html + self.options.options_map['openmp'] = ['qopenmp'] # -xSSE2 is not supported by Intel oneAPI compilers, # so use -march=x86-64 -mtune=generic when using optarch=GENERIC diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index d80d41c788..f6243bd3b3 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -2036,7 +2036,7 @@ def test_independence(self): 'CrayIntel': "-O2 -ftz -fp-speculation=safe -fp-model source -fopenmp -craype-verbose", 'GCC': "-O2 -ftree-vectorize -test -fno-math-errno -fopenmp", 'iccifort': "-O2 -test -ftz -fp-speculation=safe -fp-model source -fopenmp", - 'intel-compilers': "-O2 -test -ftz -fp-speculation=safe -fp-model precise -fiopenmp", + 'intel-compilers': "-O2 -test -ftz -fp-speculation=safe -fp-model precise -qopenmp", } toolchains = [ From 534b02987c29df66ee743ee827336ec9f83ee5bc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Nov 2023 19:29:47 +0100 Subject: [PATCH 193/312] rename 'shell' option in run_shell_cmd to 'use_bash' --- easybuild/tools/modules.py | 4 ++-- easybuild/tools/run.py | 9 ++++++--- test/framework/run.py | 19 +++++++++++++++++++ 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 60f2929a83..9c10f057d3 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -820,7 +820,7 @@ def run_module(self, *args, **kwargs): cmd_list = self.compose_cmd_list(args) cmd = ' '.join(cmd_list) # note: module commands are always run in dry mode, and are kept hidden in trace and dry run output - res = run_shell_cmd(cmd_list, env=environ, fail_on_error=False, shell=False, split_stderr=True, + res = run_shell_cmd(cmd_list, env=environ, fail_on_error=False, use_bash=False, split_stderr=True, hidden=True, in_dry_run=True, output_file=False) # stdout will contain python code (to change environment etc) @@ -1422,7 +1422,7 @@ def update(self): cmd = ' '.join(cmd_list) self.log.debug("Running command '%s'...", cmd) - res = run_shell_cmd(cmd_list, env=os.environ, fail_on_error=False, shell=False, split_stderr=True, + res = run_shell_cmd(cmd_list, env=os.environ, fail_on_error=False, use_bash=False, split_stderr=True, hidden=True) stdout, stderr = res.output, res.stderr diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 589528a6ab..040e20f76e 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -180,7 +180,7 @@ def cache_aware_func(cmd, *args, **kwargs): @run_shell_cmd_cache def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, - hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, shell=True, + hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, use_bash=True, output_file=True, stream_output=False, asynchronous=False, with_hooks=True, qa_patterns=None, qa_wait_patterns=None): """ @@ -194,7 +194,7 @@ def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=N :param in_dry_run: also run command in dry run mode :param verbose_dry_run: show that command is run in dry run mode (overrules 'hidden') :param work_dir: working directory to run command in (current working directory if None) - :param shell: execute command through bash shell (enabled by default) + :param use_bash: execute command through bash shell (enabled by default) :param output_file: collect command output in temporary output file :param stream_output: stream command output to stdout :param asynchronous: run command asynchronously @@ -270,7 +270,10 @@ def to_cmd_str(cmd): # use bash as shell instead of the default /bin/sh used by subprocess.run # (which could be dash instead of bash, like on Ubuntu, see https://wiki.ubuntu.com/DashAsBinSh) # stick to None (default value) when not running command via a shell - executable = '/bin/bash' if shell else None + if use_bash: + executable, shell = '/bin/bash', True + else: + executable, shell = None, False stderr = subprocess.PIPE if split_stderr else subprocess.STDOUT diff --git a/test/framework/run.py b/test/framework/run.py index 948eac51b1..b5d6aa9383 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1066,6 +1066,25 @@ def test_run_cmd_script(self): self.assertEqual(ec, 0) self.assertEqual(out, "hello\n") + def test_run_shell_cmd_no_bash(self): + """Testing use of run_shell_cmd with use_bash=False to call external scripts""" + py_test_script = os.path.join(self.test_prefix, 'test.py') + write_file(py_test_script, '\n'.join([ + '#!%s' % sys.executable, + 'print("hello")', + ])) + adjust_permissions(py_test_script, stat.S_IXUSR) + + with self.mocked_stdout_stderr(): + res = run_shell_cmd(py_test_script) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, "hello\n") + + with self.mocked_stdout_stderr(): + res = run_shell_cmd([py_test_script], use_bash=False) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, "hello\n") + def test_run_cmd_stream(self): """Test use of run_cmd with streaming output.""" self.mock_stdout(True) From 757b8ee60ebbeaf50e868f430a2f0f399bcadcd7 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Sat, 28 Oct 2023 20:55:49 +0200 Subject: [PATCH 194/312] Adapt module function check for EnvironmentModules Default check_module_function tests that module command is called from module shell function. With EnvironmentModules v4+, module command is usually called from the _module_raw shell function. This commit adds a specific version of check_module_function for EnvironmentModules class. Module command is first checked within _module_raw shell function definition. If not found, default test (that checks module function) is run. Add new unit test "test_environment_modules_specific" to specifically check module command definition with EnvironmentModules. Fixes #4368 --- easybuild/tools/modules.py | 25 +++++++++++++++++++++++++ easybuild/tools/run.py | 1 + test/framework/modulestool.py | 19 ++++++++++++++++++- 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 386643e706..c5789f40eb 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1328,6 +1328,31 @@ class EnvironmentModules(EnvironmentModulesTcl): MAX_VERSION = None VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d\S*)\s' + def check_module_function(self, allow_mismatch=False, regex=None): + """Check whether selected module tool matches 'module' function definition.""" + # Modules 5.1.0+: module command is called from _module_raw shell function + # Modules 4.2.0..5.0.1: module command is called from _module_raw shell function if it has + # been initialized in an interactive shell session (i.e., a session attached to a tty) + if self.testing: + if '_module_raw' in os.environ: + out, ec = os.environ['_module_raw'], 0 + else: + out, ec = None, 1 + else: + cmd = "type _module_raw" + out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, force_in_dry_run=True, trace=False) + + if regex is None: + regex = r".*%s" % os.path.basename(self.cmd) + mod_cmd_re = re.compile(regex, re.M) + + if ec == 0 and mod_cmd_re.search(out): + self.log.debug("Found pattern '%s' in defined '_module_raw' function." % mod_cmd_re.pattern) + else: + self.log.debug("Pattern '%s' not found in '_module_raw' function, falling back to 'module' function", + mod_cmd_re.pattern) + super(EnvironmentModules, self).check_module_function(allow_mismatch, regex) + def check_module_output(self, cmd, stdout, stderr): """Check output of 'module' command, see if if is potentially invalid.""" if "_mlstatus = False" in stdout: diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 8916d80795..08b109ec15 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -71,6 +71,7 @@ "sysctl -n machdep.cpu.brand_string", # used in get_cpu_model (OS X) "sysctl -n machdep.cpu.vendor", # used in get_cpu_vendor (OS X) "type module", # used in ModulesTool.check_module_function + "type _module_raw", # used in EnvironmentModules.check_module_function "ulimit -u", # used in det_parallelism ] diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index 59c6872b93..fb991ad797 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -39,7 +39,7 @@ from easybuild.tools import modules, StrictVersion from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file, which, write_file -from easybuild.tools.modules import Lmod +from easybuild.tools.modules import EnvironmentModules, Lmod from test.framework.utilities import init_config @@ -192,6 +192,23 @@ def test_lmod_specific(self): # test updating local spider cache (but don't actually update the local cache file!) self.assertTrue(lmod.update(), "Updated local Lmod spider cache is non-empty") + def test_environment_modules_specific(self): + """Environment Modules-specific test (skipped unless installed).""" + modulecmd_abspath = which(EnvironmentModules.COMMAND) + # only run this test if 'modulecmd.tcl' is installed + if modulecmd_abspath is not None: + # redefine 'module' and '_module_raw' function (deliberate mismatch with used module + # command in EnvironmentModules) + os.environ['_module_raw'] = "() { eval `/usr/share/Modules/libexec/foo.tcl' bash $*`;\n}" + os.environ['module'] = "() { _module_raw \"$@\" 2>&1;\n}" + error_regex = ".*pattern .* not found in defined 'module' function" + self.assertErrorRegex(EasyBuildError, error_regex, EnvironmentModules, testing=True) + + # redefine '_module_raw' function with correct module command + os.environ['_module_raw'] = "() { eval `/usr/share/Modules/libexec/modulecmd.tcl' bash $*`;\n}" + mt = EnvironmentModules(testing=True) + self.assertIsInstance(mt.loaded_modules(), list) # dummy usage + def tearDown(self): """Testcase cleanup.""" super(ModulesToolTest, self).tearDown() From 0288ca35420fe54a298e547e17f1d057c96b2099 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Fri, 10 Nov 2023 13:56:14 +0100 Subject: [PATCH 195/312] fix LIBBLAS_MT for flexiblas, ensure -lpthread is included --- easybuild/toolchains/linalg/flexiblas.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/toolchains/linalg/flexiblas.py b/easybuild/toolchains/linalg/flexiblas.py index 43b450b3de..c266aff248 100644 --- a/easybuild/toolchains/linalg/flexiblas.py +++ b/easybuild/toolchains/linalg/flexiblas.py @@ -70,6 +70,7 @@ class FlexiBLAS(LinAlg): """ BLAS_MODULE_NAME = ['FlexiBLAS'] BLAS_LIB = ['flexiblas'] + BLAS_LIB_MT = ['flexiblas'] BLAS_INCLUDE_DIR = [os.path.join('include', 'flexiblas')] BLAS_FAMILY = TC_CONSTANT_FLEXIBLAS From 047b837597f350265c2d7ba6b48b400dc1b780cf Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Nov 2023 11:21:34 +0100 Subject: [PATCH 196/312] print error message when EasyBuildError was raised in main_with_hooks function --- easybuild/main.py | 2 +- easybuild/tools/build_log.py | 14 ++++++++++++-- test/framework/toy_build.py | 23 +++++++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 0f2c69733c..612d49f4f0 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -754,7 +754,7 @@ def main_with_hooks(args=None): main(args=args, prepared_cfg_data=(init_session_state, eb_go, cfg_settings)) except EasyBuildError as err: run_hook(FAIL, hooks, args=[err]) - sys.exit(1) + print_error(err.msg, exit_on_error=True, exit_code=1) except KeyboardInterrupt as err: run_hook(CANCEL, hooks, args=[err]) print_error("Cancelled by user: %s" % err) diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index e8a479e24e..f8529fff57 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -335,8 +335,18 @@ def print_error(msg, *args, **kwargs): if args: msg = msg % args + # grab exit code, if specified; + # also consider deprecated 'exitCode' option + exitCode = kwargs.pop('exitCode', None) + exit_code = kwargs.pop('exit_code', exitCode) + if exitCode is not None: + _init_easybuildlog.deprecated("'exitCode' option in print_error function is replaced with 'exit_code'", '6.0') + + # use 1 as defaut exit code + if exit_code is None: + exit_code = 1 + log = kwargs.pop('log', None) - exitCode = kwargs.pop('exitCode', 1) opt_parser = kwargs.pop('opt_parser', None) exit_on_error = kwargs.pop('exit_on_error', True) silent = kwargs.pop('silent', False) @@ -348,7 +358,7 @@ def print_error(msg, *args, **kwargs): if opt_parser: opt_parser.print_shorthelp() sys.stderr.write("ERROR: %s\n" % msg) - sys.exit(exitCode) + sys.exit(exit_code) elif log is not None: raise EasyBuildError(msg) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 74a467175b..40384d7582 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -4104,6 +4104,29 @@ def pre_configure_hook(self, *args, **kwargs): stderr = stderr.getvalue() self.assertTrue(regex.search(stderr), f"Pattern '{regex.pattern}' should be found in {stderr}") + def test_eb_error(self): + """ + Test whether main function as run by 'eb' command print error messages to stderr. + """ + topdir = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + toy_ec = os.path.join(topdir, 'test', 'framework', 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = read_file(toy_ec) + test_ec_txt += "\ndependencies = [('nosuchdep', '1.0')]" + write_file(test_ec, test_ec_txt) + + with self.mocked_stdout_stderr() as (_, stderr): + cleanup() + try: + main_with_hooks(args=[test_ec, '--robot', '--force']) + except SystemExit: + pass + + regex = re.compile("^ERROR: Missing dependencies", re.M) + stderr = stderr.getvalue() + self.assertTrue(regex.search(stderr), f"Pattern '{regex.pattern}' should be found in {stderr}") + def suite(): """ return all the tests in this file """ From a3f38735455d5a8012120b9b35de2fa10dae7fe9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 16 Nov 2023 10:32:32 +0100 Subject: [PATCH 197/312] start adopting `run_shell_cmd` in easyblock.py (+ use f-strings) --- easybuild/framework/easyblock.py | 124 +++++++++++++++---------------- 1 file changed, 61 insertions(+), 63 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 5886fe8500..52a700bdfe 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -87,7 +87,7 @@ from easybuild.tools.hooks import MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP from easybuild.tools.hooks import POSTPROC_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP from easybuild.tools.hooks import SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook -from easybuild.tools.run import RunShellCmdError, check_async_cmd, run_cmd +from easybuild.tools.run import RunShellCmdError, check_async_cmd, run_cmd, run_shell_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version @@ -1780,21 +1780,20 @@ def skip_extensions_sequential(self, exts_filter): exts_cnt = len(self.ext_instances) - res = [] + exts = [] for idx, ext_inst in enumerate(self.ext_instances): cmd, stdin = resolve_exts_filter_template(exts_filter, ext_inst) - (out, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, - regexp=False, trace=False) - self.log.info("exts_filter result for %s: exit code %s; output: %s", ext_inst.name, ec, out) - if ec == 0: - print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log) + res = run_shell_cmd(cmd, stdin=stdin, fail_on_error=False, hidden=True) + self.log.info(f"exts_filter result for {ext_inst.name}: exit code {res.exit_code}; output: {res.output}") + if res.exit_code == 0: + print_msg(f"skipping extension {ext_inst.name}", silent=self.silent, log=self.log) else: - self.log.info("Not skipping %s", ext_inst.name) - res.append(ext_inst) + self.log.info(f"Not skipping {ext_inst.name}") + exts.append(ext_inst) - self.update_exts_progress_bar("skipping installed extensions (%d/%d checked)" % (idx + 1, exts_cnt)) + self.update_exts_progress_bar(f"skipping installed extensions ({idx + 1}/{exts_cnt} checked)") - self.ext_instances = res + self.ext_instances = exts self.update_exts_progress_bar("already installed extensions filtered out", total=len(self.ext_instances)) def skip_extensions_parallel(self, exts_filter): @@ -2714,17 +2713,15 @@ def test_step(self): """Run unit tests provided by software (if any).""" unit_test_cmd = self.cfg['runtest'] if unit_test_cmd: - - self.log.debug("Trying to execute %s as a command for running unit tests...", unit_test_cmd) - (out, _) = run_cmd(unit_test_cmd, log_all=True, simple=False) - - return out + self.log.debug(f"Trying to execute {unit_test_cmd} as a command for running unit tests...") + res = run_shell_cmd(unit_test_cmd) + return res.output def _test_step(self): """Run the test_step and handles failures""" try: self.test_step() - except EasyBuildError as err: + except RunShellCmdError as err: self.report_test_failure(err) def stage_install_step(self): @@ -2977,17 +2974,17 @@ def run_post_install_commands(self, commands=None): commands = self.cfg['postinstallcmds'] if commands: - self.log.debug("Specified post install commands: %s", commands) + self.log.debug(f"Specified post install commands: {commands}") # make sure we have a list of commands if not isinstance(commands, (list, tuple)): - error_msg = "Invalid value for 'postinstallcmds', should be list or tuple of strings: %s" - raise EasyBuildError(error_msg, commands) + error_msg = f"Invalid value for 'postinstallcmds', should be list or tuple of strings: {commands}" + raise EasyBuildError(error_msg) for cmd in commands: if not isinstance(cmd, str): - raise EasyBuildError("Invalid element in 'postinstallcmds', not a string: %s", cmd) - run_cmd(cmd, simple=True, log_ok=True, log_all=True) + raise EasyBuildError(f"Invalid element in 'postinstallcmds', not a string: {cmd}") + run_shell_cmd(cmd) def apply_post_install_patches(self, patches=None): """ @@ -3104,8 +3101,10 @@ def sanity_check_rpath(self, rpath_dirs=None, check_readelf_rpath=True): # hard reset $LD_LIBRARY_PATH before running RPATH sanity check orig_env = env.unset_env_vars(['LD_LIBRARY_PATH']) - self.log.debug("$LD_LIBRARY_PATH during RPATH sanity check: %s", os.getenv('LD_LIBRARY_PATH', '(empty)')) - self.log.debug("List of loaded modules: %s", self.modules_tool.list()) + ld_library_path = os.getenv('LD_LIBRARY_PATH', '(empty)') + self.log.debug(f"$LD_LIBRARY_PATH during RPATH sanity check: {ld_library_path}") + modules_list = self.modules_tool.list() + self.log.debug(f"List of loaded modules: {modules_list}") not_found_regex = re.compile(r'(\S+)\s*\=\>\s*not found') readelf_rpath_regex = re.compile('(RPATH)', re.M) @@ -3114,33 +3113,31 @@ def sanity_check_rpath(self, rpath_dirs=None, check_readelf_rpath=True): # For example, libcuda.so.1 should never be RPATH-ed by design, # see https://github.com/easybuilders/easybuild-framework/issues/4095 filter_rpath_sanity_libs = build_option('filter_rpath_sanity_libs') - msg = "Ignoring the following libraries if they are not found by RPATH sanity check: %s" - self.log.info(msg, filter_rpath_sanity_libs) + msg = "Ignoring the following libraries if they are not found by RPATH sanity check: {filter_rpath_sanity_libs}" + self.log.info(msg) if rpath_dirs is None: rpath_dirs = self.cfg['bin_lib_subdirs'] or self.bin_lib_subdirs() if not rpath_dirs: rpath_dirs = DEFAULT_BIN_LIB_SUBDIRS - self.log.info("Using default subdirectories for binaries/libraries to verify RPATH linking: %s", - rpath_dirs) + self.log.info(f"Using default subdirs for binaries/libraries to verify RPATH linking: {rpath_dirs}") else: - self.log.info("Using specified subdirectories for binaries/libraries to verify RPATH linking: %s", - rpath_dirs) + self.log.info(f"Using specified subdirs for binaries/libraries to verify RPATH linking: {rpath_dirs}") for dirpath in [os.path.join(self.installdir, d) for d in rpath_dirs]: if os.path.exists(dirpath): - self.log.debug("Sanity checking RPATH for files in %s", dirpath) + self.log.debug(f"Sanity checking RPATH for files in {dirpath}") for path in [os.path.join(dirpath, x) for x in os.listdir(dirpath)]: - self.log.debug("Sanity checking RPATH for %s", path) + self.log.debug(f"Sanity checking RPATH for {path}") out = get_linked_libs_raw(path) if out is None: - msg = "Failed to determine dynamically linked libraries for %s, " + msg = "Failed to determine dynamically linked libraries for {path}, " msg += "so skipping it in RPATH sanity check" - self.log.debug(msg, path) + self.log.debug(msg) else: # check whether all required libraries are found via 'ldd' matches = re.findall(not_found_regex, out) @@ -3148,34 +3145,34 @@ def sanity_check_rpath(self, rpath_dirs=None, check_readelf_rpath=True): # For each match, check if the library is in the exception list for match in matches: if match in filter_rpath_sanity_libs: - msg = "Library %s not found for %s, but ignored " - msg += "since it is on the rpath exception list: %s" - self.log.info(msg, match, path, filter_rpath_sanity_libs) + msg = f"Library {match} not found for {path}, but ignored " + msg += f"since it is on the rpath exception list: {filter_rpath_sanity_libs}" + self.log.info(msg) else: - fail_msg = "Library %s not found for %s" % (match, path) + fail_msg = f"Library {match} not found for {path}" self.log.warning(fail_msg) fails.append(fail_msg) else: - self.log.debug("Output of 'ldd %s' checked, looks OK", path) + self.log.debug(f"Output of 'ldd {path}' checked, looks OK") # check whether RPATH section in 'readelf -d' output is there if check_readelf_rpath: fail_msg = None - out, ec = run_cmd("readelf -d %s" % path, simple=False, trace=False) - if ec: - fail_msg = "Failed to run 'readelf %s': %s" % (path, out) - elif not readelf_rpath_regex.search(out): - fail_msg = "No '(RPATH)' found in 'readelf -d' output for %s: %s" % (path, out) + res = run_shell_cmd(f"readelf -d {path}", fail_on_error=False) + if res.exit_code: + fail_msg = f"Failed to run 'readelf -d {path}': {res.output}" + elif not readelf_rpath_regex.search(res.output): + fail_msg = f"No '(RPATH)' found in 'readelf -d' output for {path}: {out}" if fail_msg: self.log.warning(fail_msg) fails.append(fail_msg) else: - self.log.debug("Output of 'readelf -d %s' checked, looks OK", path) + self.log.debug(f"Output of 'readelf -d {path}' checked, looks OK") else: self.log.debug("Skipping the RPATH section check with 'readelf -d', as requested") else: - self.log.debug("Not sanity checking files in non-existing directory %s", dirpath) + self.log.debug(f"Not sanity checking files in non-existing directory {dirpath}") env.restore_env_vars(orig_env) @@ -3604,19 +3601,20 @@ def xs2str(xs): change_dir(self.installdir) # run sanity check commands - for command in commands: + for cmd in commands: - trace_msg("running command '%s' ..." % command) + trace_msg(f"running command '{cmd}' ...") - out, ec = run_cmd(command, simple=False, log_ok=False, log_all=False, trace=False) - if ec != 0: - fail_msg = "sanity check command %s exited with code %s (output: %s)" % (command, ec, out) + res = run_shell_cmd(cmd, fail_on_error=False, hidden=True) + if res.exit_code != 0: + fail_msg = f"sanity check command {cmd} exited with code {res.exit_code} (output: {res.output})" self.sanity_check_fail_msgs.append(fail_msg) - self.log.warning("Sanity check: %s" % self.sanity_check_fail_msgs[-1]) + self.log.warning(f"Sanity check: {fail_msg}") else: - self.log.info("sanity check command %s ran successfully! (output: %s)" % (command, out)) + self.log.info(f"sanity check command {cmd} ran successfully! (output: {res.output})") - trace_msg("result for command '%s': %s" % (command, ('FAILED', 'OK')[ec == 0])) + cmd_result_str = ('FAILED', 'OK')[res.exit_code == 0] + trace_msg(f"result for command '{cmd}': {cmd_result_str}") # also run sanity check for extensions (unless we are an extension ourselves) if not extension: @@ -3643,7 +3641,7 @@ def xs2str(xs): # pass or fail if self.sanity_check_fail_msgs: - raise EasyBuildError("Sanity check failed: %s", '\n'.join(self.sanity_check_fail_msgs)) + raise EasyBuildError("Sanity check failed: " + '\n'.join(self.sanity_check_fail_msgs)) else: self.log.debug("Sanity check passed!") @@ -3854,20 +3852,20 @@ def test_cases_step(self): for test in self.cfg['tests']: change_dir(self.orig_workdir) if os.path.isabs(test): - path = test + test_cmd = test else: for source_path in source_paths(): - path = os.path.join(source_path, self.name, test) - if os.path.exists(path): + test_cmd = os.path.join(source_path, self.name, test) + if os.path.exists(test_cmd): break - if not os.path.exists(path): - raise EasyBuildError("Test specifies invalid path: %s", path) + if not os.path.exists(test_cmd): + raise EasyBuildError(f"Test specifies invalid path: {test_cmd}") try: - self.log.debug("Running test %s" % path) - run_cmd(path, log_all=True, simple=True) + self.log.debug(f"Running test {test_cmd}") + run_shell_cmd(test_cmd) except EasyBuildError as err: - raise EasyBuildError("Running test %s failed: %s", path, err) + raise EasyBuildError(f"Running test {test_cmd} failed: {err}") def update_config_template_run_step(self): """Update the the easyconfig template dictionary with easyconfig.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP names""" From 2ce5868740716f27bbd092bbeeaef1b67475cca4 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 21 Nov 2023 18:32:28 +0800 Subject: [PATCH 198/312] relax major version match regex in find_related_easyconfigs --- easybuild/framework/easyconfig/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index abe8f998ea..c6649c383b 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -472,7 +472,7 @@ def find_related_easyconfigs(path, ec): if len(parsed_version) >= 2: version_patterns.append(r'%s\.%s\.\w+' % tuple(parsed_version[:2])) # major/minor version match if parsed_version != parsed_version[0]: - version_patterns.append(r'%s\.[\d-]+\.\w+' % parsed_version[0]) # major version match + version_patterns.append(r'%s\.[\d-]+(\.\w+)*' % parsed_version[0]) # major version match version_patterns.append(r'[\w.]+') # any version regexes = [] From 95e8c480b23f8e653957d0d8dc646fb877ebb1b7 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 21 Nov 2023 19:52:49 +0000 Subject: [PATCH 199/312] When combining multi_deps with iterative builds, it generates a list with duplicates. This commit fixes that --- easybuild/tools/module_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index aaf97d3194..0da350c1bd 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -666,7 +666,7 @@ def _generate_help_text(self): if multi_deps: compatible_modules_txt = '\n'.join([ "This module is compatible with the following modules, one of each line is required:", - ] + ['* %s' % d for d in multi_deps]) + ] + ['* %s' % d for d in set(multi_deps)]) lines.extend(self._generate_section("Compatible modules", compatible_modules_txt)) # Extensions (if any) From 3714b8cf30f2cb86af9e0d1b292cac97997a3769 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Wed, 22 Nov 2023 15:15:17 +0000 Subject: [PATCH 200/312] Use nub instead of set to preserve order --- easybuild/tools/module_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 0da350c1bd..c65879009e 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -49,7 +49,7 @@ from easybuild.tools.filetools import convert_name, mkdir, read_file, remove_file, resolve_path, symlink, write_file from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, EnvironmentModulesC, Lmod, modules_tool from easybuild.tools.py2vs3 import string_type -from easybuild.tools.utilities import get_subclasses, quote_str +from easybuild.tools.utilities import get_subclasses, nub, quote_str _log = fancylogger.getLogger('module_generator', fname=False) @@ -666,7 +666,7 @@ def _generate_help_text(self): if multi_deps: compatible_modules_txt = '\n'.join([ "This module is compatible with the following modules, one of each line is required:", - ] + ['* %s' % d for d in set(multi_deps)]) + ] + ['* %s' % d for d in nub(multi_deps)]) lines.extend(self._generate_section("Compatible modules", compatible_modules_txt)) # Extensions (if any) From a3a427bf76b06c6ab6ff88fcc8a500d2eda38449 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 25 Nov 2023 09:54:36 +0000 Subject: [PATCH 201/312] detect Fortran .mod files in GCCcore installations --- easybuild/framework/easyblock.py | 21 +++++++++++++++++++++ easybuild/tools/config.py | 1 + easybuild/tools/options.py | 1 + 3 files changed, 23 insertions(+) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 52a700bdfe..7d31953520 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3303,6 +3303,19 @@ def regex_for_lib(lib): return fail_msg + def sanity_check_mod_files(self): + """ + Check installation for Fortran .mod files + """ + self.log.debug("Check for .mod files in install directory") + mod_files = glob.glob(os.path.join(self.installdir, '**', '*.mod'), recursive=True) + + fail_msg = None + if mod_files: + fail_msg = ".mod files (%s) found in the installation." % ', '.join(mod_files) + + return fail_msg + def _sanity_check_step_common(self, custom_paths, custom_commands): """ Determine sanity check paths and commands to use. @@ -3625,6 +3638,14 @@ def xs2str(xs): self.log.warning("Check for required/banned linked shared libraries failed!") self.sanity_check_fail_msgs.append(linked_shared_lib_fails) + if self.toolchain.name in ['GCCcore']: + mod_files_found = self.sanity_check_mod_files() + if mod_files_found: + if build_option('fail_on_mod_files'): + self.sanity_check_fail_msgs.append(mod_files_found) + else: + print_warning(mod_files_found) + # cleanup if self.fake_mod_data: self.clean_up_fake_module(self.fake_mod_data) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index b18bae480c..c5db28b05f 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -277,6 +277,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'enforce_checksums', 'experimental', 'extended_dry_run', + 'fail_on_mod_files', 'force', 'generate_devel_module', 'group_writable_installdir', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 999b62bb5c..faa1f48e80 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -400,6 +400,7 @@ def override_options(self): None, 'store_true', False), 'extra-modules': ("List of extra modules to load after setting up the build environment", 'strlist', 'extend', None), + 'fail-on-mod-files': ("Fail if .mod files are detected in a GCCcore install", None, 'store_true', False), 'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, " "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False), 'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, " From f148ffbf6e67fa49e13cc9f317d21e7576678264 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 25 Nov 2023 10:19:40 +0000 Subject: [PATCH 202/312] easyconfig param to skip check --- easybuild/framework/easyconfig/default.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index dd91229d1e..74b96cffde 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -130,6 +130,7 @@ 'sanity_check_paths': [{}, ("List of files and directories to check " "(format: {'files':, 'dirs':})"), BUILD], 'skip': [False, "Skip existing software", BUILD], + 'skip_mod_files_check': [False, "Skip the check for .mod files in a GCCcore level install", BUILD], 'skipsteps': [[], "Skip these steps", BUILD], 'source_urls': [[], "List of URLs for source files", BUILD], 'sources': [[], "List of source files", BUILD], From ec6ece644c7b87d43515c3317d1e0e1688088e60 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 25 Nov 2023 10:46:55 +0000 Subject: [PATCH 203/312] skip check when not needed --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 7d31953520..f789ffea2b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3638,7 +3638,7 @@ def xs2str(xs): self.log.warning("Check for required/banned linked shared libraries failed!") self.sanity_check_fail_msgs.append(linked_shared_lib_fails) - if self.toolchain.name in ['GCCcore']: + if self.toolchain.name in ['GCCcore'] and not self.cfg._config['skip_mod_files_check']: mod_files_found = self.sanity_check_mod_files() if mod_files_found: if build_option('fail_on_mod_files'): From d887426f83df97dbd9c5d26bd78747cba7efb0eb Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 25 Nov 2023 11:00:12 +0000 Subject: [PATCH 204/312] correctly check for if the parameter is set --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f789ffea2b..a57db2591a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3638,7 +3638,7 @@ def xs2str(xs): self.log.warning("Check for required/banned linked shared libraries failed!") self.sanity_check_fail_msgs.append(linked_shared_lib_fails) - if self.toolchain.name in ['GCCcore'] and not self.cfg._config['skip_mod_files_check']: + if self.toolchain.name in ['GCCcore'] and not self.cfg['skip_mod_files_check']: mod_files_found = self.sanity_check_mod_files() if mod_files_found: if build_option('fail_on_mod_files'): From 809b6948d5f024fe0e9e137eb5c972834a09dcf6 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 25 Nov 2023 12:52:53 +0000 Subject: [PATCH 205/312] add testing --- test/framework/toy_build.py | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 40384d7582..54582ec48e 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3955,6 +3955,44 @@ def test_toy_build_sanity_check_linked_libs(self): self._test_toy_build(ec_file=test_ec, extra_args=args, force=False, raise_error=True, verbose=False, verify=False) + def test_toy_mod_files(self): + """Check detection of .mod files""" + test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + test_ec_txt = read_file(toy_ec) + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ec_txt) + + with self.mocked_stdout_stderr(): + self._test_toy_build(ec_file=test_ec) + + test_ec_txt += "\npostinstallcmds += ['touch %(installdir)s/lib/file.mod']" + write_file(test_ec, test_ec_txt) + + with self.mocked_stdout_stderr(): + self._test_toy_build(ec_file=test_ec) + + args = ['--try-toolchain=GCCcore,6.2.0', '--disable-map-toolchains'] + self.mock_stdout(True) + self.mock_stderr(True) + self._test_toy_build(ec_file=test_ec, extra_args=args) + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + pattern = r"WARNING: .mod files (.*) found in the installation." + self.assertRegex(stderr.strip(), pattern) + + args += ['--fail-on-mod-files'] + pattern = r"Sanity check failed: .mod files (.*) found in the installation." + self.assertErrorRegex(EasyBuildError, pattern, self.run_test_toy_build_with_output, ec_file=test_ec, + extra_args=args, verify=False, fails=True, verbose=False, raise_error=True) + + test_ec_txt += "\nskip_mod_files_check = True" + write_file(test_ec, test_ec_txt) + + with self.mocked_stdout_stderr(): + self._test_toy_build(ec_file=test_ec, extra_args=args) + def test_toy_ignore_test_failure(self): """Check whether use of --ignore-test-failure is mentioned in build output.""" args = ['--ignore-test-failure'] From 0b3daf25fcb1237c39d867028d9045ebeff82aab Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 25 Nov 2023 13:03:42 +0000 Subject: [PATCH 206/312] address review comments --- easybuild/framework/easyblock.py | 14 +++++++------- easybuild/framework/easyconfig/default.py | 2 +- easybuild/tools/config.py | 2 +- easybuild/tools/options.py | 2 +- test/framework/toy_build.py | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a57db2591a..954bb3212e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3307,7 +3307,7 @@ def sanity_check_mod_files(self): """ Check installation for Fortran .mod files """ - self.log.debug("Check for .mod files in install directory") + self.log.debug(f"Checking for .mod files in install directory {self.installdir}...") mod_files = glob.glob(os.path.join(self.installdir, '**', '*.mod'), recursive=True) fail_msg = None @@ -3638,13 +3638,13 @@ def xs2str(xs): self.log.warning("Check for required/banned linked shared libraries failed!") self.sanity_check_fail_msgs.append(linked_shared_lib_fails) - if self.toolchain.name in ['GCCcore'] and not self.cfg['skip_mod_files_check']: - mod_files_found = self.sanity_check_mod_files() - if mod_files_found: - if build_option('fail_on_mod_files'): - self.sanity_check_fail_msgs.append(mod_files_found) + if self.toolchain.name in ['GCCcore'] and not self.cfg['skip_mod_files_sanity_check']: + mod_files_found_msg = self.sanity_check_mod_files() + if mod_files_found_msg: + if build_option('fail_on_mod_files_gcccore'): + self.sanity_check_fail_msgs.append(mod_files_found_msg) else: - print_warning(mod_files_found) + print_warning(mod_files_found_msg) # cleanup if self.fake_mod_data: diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 74b96cffde..5de9799e54 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -130,7 +130,7 @@ 'sanity_check_paths': [{}, ("List of files and directories to check " "(format: {'files':, 'dirs':})"), BUILD], 'skip': [False, "Skip existing software", BUILD], - 'skip_mod_files_check': [False, "Skip the check for .mod files in a GCCcore level install", BUILD], + 'skip_mod_files_sanity_check': [False, "Skip the check for .mod files in a GCCcore level install", BUILD], 'skipsteps': [[], "Skip these steps", BUILD], 'source_urls': [[], "List of URLs for source files", BUILD], 'sources': [[], "List of source files", BUILD], diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index c5db28b05f..d8e37ac0da 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -277,7 +277,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'enforce_checksums', 'experimental', 'extended_dry_run', - 'fail_on_mod_files', + 'fail_on_mod_files_gcccore', 'force', 'generate_devel_module', 'group_writable_installdir', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index faa1f48e80..766b891bda 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -400,7 +400,7 @@ def override_options(self): None, 'store_true', False), 'extra-modules': ("List of extra modules to load after setting up the build environment", 'strlist', 'extend', None), - 'fail-on-mod-files': ("Fail if .mod files are detected in a GCCcore install", None, 'store_true', False), + 'fail-on-mod-files-gcccore': ("Fail if .mod files are detected in a GCCcore install", None, 'store_true', False), 'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, " "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False), 'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, " diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 54582ec48e..9bf85607fa 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3982,12 +3982,12 @@ def test_toy_mod_files(self): pattern = r"WARNING: .mod files (.*) found in the installation." self.assertRegex(stderr.strip(), pattern) - args += ['--fail-on-mod-files'] + args += ['--fail-on-mod-files-gcccore'] pattern = r"Sanity check failed: .mod files (.*) found in the installation." self.assertErrorRegex(EasyBuildError, pattern, self.run_test_toy_build_with_output, ec_file=test_ec, extra_args=args, verify=False, fails=True, verbose=False, raise_error=True) - test_ec_txt += "\nskip_mod_files_check = True" + test_ec_txt += "\nskip_mod_files_sanity_check = True" write_file(test_ec, test_ec_txt) with self.mocked_stdout_stderr(): From 1644c930c5126edd73311395d313d4b0ddb4b109 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 25 Nov 2023 13:04:48 +0000 Subject: [PATCH 207/312] appease the hound --- easybuild/tools/options.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 766b891bda..1d0330ee30 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -400,7 +400,8 @@ def override_options(self): None, 'store_true', False), 'extra-modules': ("List of extra modules to load after setting up the build environment", 'strlist', 'extend', None), - 'fail-on-mod-files-gcccore': ("Fail if .mod files are detected in a GCCcore install", None, 'store_true', False), + 'fail-on-mod-files-gcccore': ("Fail if .mod files are detected in a GCCcore install", None, 'store_true', + False), 'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, " "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False), 'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, " From 871f955ca0317a3d5d1bcd2f5955b02d445ac62d Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Sun, 26 Nov 2023 22:01:02 +0100 Subject: [PATCH 208/312] Add formating as json --- easybuild/tools/docs.py | 30 ++++++++++++++++++++++++++++++ easybuild/tools/options.py | 4 ++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 4f4bfca99c..84666a6e10 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -39,6 +39,7 @@ import copy import inspect import os +import json from easybuild.tools import LooseVersion from easybuild.base import fancylogger @@ -75,6 +76,7 @@ FORMAT_MD = 'md' FORMAT_RST = 'rst' FORMAT_TXT = 'txt' +FORMAT_JSON = 'json' def generate_doc(name, params): @@ -1024,6 +1026,34 @@ def list_software_txt(software, detailed=False): return '\n'.join(lines) +def list_software_json(software, detailed=False): + """ + Return overview of supported software in json + + :param software: software information (strucuted like list_software does) + :param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info) + :return: multi-line string presenting requested info + """ + lines = ['['] + for key in sorted(software, key=lambda x: x.lower()): + for tmp in software[key]: + if detailed: + # deep copy here to avoid modifying the original dict + x = copy.deepcopy(tmp) + x['description'] = x['description'].split('\n')[0].strip() + else: + x = {} + x['name'] = key + + lines.append(json.dumps(x, indent=4) + ",") + if detailed: + break + # remove last comma + if len(lines) > 1: + lines[-1] = lines[-1][:-1] + return '\n'.join(lines) + '\n]' + + def list_toolchains(output_format=FORMAT_TXT): """Show list of known toolchains.""" _, all_tcs = search_toolchain('') diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index e69d96b2f6..2a7d990e01 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -79,7 +79,7 @@ from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path from easybuild.tools.config import BuildOptions, ConfigurationVariables from easybuild.tools.configobj import ConfigObj, ConfigObjError -from easybuild.tools.docs import FORMAT_MD, FORMAT_RST, FORMAT_TXT +from easybuild.tools.docs import FORMAT_MD, FORMAT_RST, FORMAT_TXT, FORMAT_JSON from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses from easybuild.tools.docs import avail_toolchain_opts, avail_easyconfig_params, avail_easyconfig_templates from easybuild.tools.docs import list_easyblocks, list_toolchains @@ -469,7 +469,7 @@ def override_options(self): 'mpi-tests': ("Run MPI tests (when relevant)", None, 'store_true', True), 'optarch': ("Set architecture optimization, overriding native architecture optimizations", None, 'store', None), - 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, [FORMAT_MD, FORMAT_RST, FORMAT_TXT]), + 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, [FORMAT_MD, FORMAT_RST, FORMAT_TXT, FORMAT_JSON]), 'output-style': ("Control output style; auto implies using Rich if available to produce rich output, " "with fallback to basic colored output", 'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES), From c3bde441c3710f3dbcd8ab873385795d3ff3179e Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Sun, 26 Nov 2023 22:26:14 +0100 Subject: [PATCH 209/312] Add json list software test --- easybuild/tools/docs.py | 2 +- easybuild/tools/options.py | 3 +- test/framework/docs.py | 88 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 84666a6e10..74684842dd 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1040,7 +1040,7 @@ def list_software_json(software, detailed=False): if detailed: # deep copy here to avoid modifying the original dict x = copy.deepcopy(tmp) - x['description'] = x['description'].split('\n')[0].strip() + x['description'] = ' '.join(x['description'].split('\n')).strip() else: x = {} x['name'] = key diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 2a7d990e01..a1f9d756eb 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -469,7 +469,8 @@ def override_options(self): 'mpi-tests': ("Run MPI tests (when relevant)", None, 'store_true', True), 'optarch': ("Set architecture optimization, overriding native architecture optimizations", None, 'store', None), - 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, [FORMAT_MD, FORMAT_RST, FORMAT_TXT, FORMAT_JSON]), + 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, + [FORMAT_MD, FORMAT_RST, FORMAT_TXT, FORMAT_JSON]), 'output-style': ("Control output style; auto implies using Rich if available to produce rich output, " "with fallback to basic colored output", 'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES), diff --git a/test/framework/docs.py b/test/framework/docs.py index 84d862bd3c..829369934a 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -405,6 +405,91 @@ ``1.4``|``GCC/4.6.3``, ``system`` ``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} +LIST_SOFTWARE_SIMPLE_MD = """# List of supported software + +EasyBuild supports 2 different software packages (incl. toolchains, bundles): + +[g](#g) + + +## G + +* GCC +* gzip""" + + + +LIST_SOFTWARE_DETAILED_MD = """# List of supported software + +EasyBuild supports 2 different software packages (incl. toolchains, bundles): + +[g](#g) + + +## G + + +[GCC](#gcc) - [gzip](#gzip) + + +### GCC + +%(gcc_descr)s + +*homepage*: + +version |toolchain +---------|---------- +``4.6.3``|``system`` + +### gzip + +%(gzip_descr)s + +*homepage*: + +version|toolchain +-------|------------------------------- +``1.4``|``GCC/4.6.3``, ``system`` +``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} + +LIST_SOFTWARE_SIMPLE_JSON = """[ +{ + "name": "GCC" +}, +{ + "name": "gzip" +}, +{ + "name": "gzip" +}, +{ + "name": "gzip" +}, +{ + "name": "gzip" +} +]""" + +LIST_SOFTWARE_DETAILED_JSON = """[ +{ + "toolchain": "system", + "description": "%(gcc_descr)s", + "homepage": "http://gcc.gnu.org/", + "version": "4.6.3", + "versionsuffix": "", + "name": "GCC" +}, +{ + "toolchain": "GCC/4.6.3", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.4", + "versionsuffix": "", + "name": "gzip" +} +]""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} + class DocsTest(EnhancedTestCase): @@ -587,6 +672,9 @@ def test_list_software(self): self.assertEqual(list_software(output_format='md'), LIST_SOFTWARE_SIMPLE_MD) self.assertEqual(list_software(output_format='md', detailed=True), LIST_SOFTWARE_DETAILED_MD) + self.assertEqual(list_software(output_format='json'), LIST_SOFTWARE_SIMPLE_JSON) + self.assertEqual(list_software(output_format='json', detailed=True), LIST_SOFTWARE_DETAILED_JSON) + # GCC/4.6.3 is installed, no gzip module installed txt = list_software(output_format='txt', detailed=True, only_installed=True) self.assertTrue(re.search(r'^\* GCC', txt, re.M)) From 1cc58f9971498c4b175b34937dc061568cea9900 Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Sun, 26 Nov 2023 22:27:50 +0100 Subject: [PATCH 210/312] Fix format... --- easybuild/tools/options.py | 2 +- test/framework/docs.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index a1f9d756eb..64c46b61b5 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -469,7 +469,7 @@ def override_options(self): 'mpi-tests': ("Run MPI tests (when relevant)", None, 'store_true', True), 'optarch': ("Set architecture optimization, overriding native architecture optimizations", None, 'store', None), - 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, + 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, [FORMAT_MD, FORMAT_RST, FORMAT_TXT, FORMAT_JSON]), 'output-style': ("Control output style; auto implies using Rich if available to produce rich output, " "with fallback to basic colored output", diff --git a/test/framework/docs.py b/test/framework/docs.py index 829369934a..5894796b05 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -417,8 +417,6 @@ * GCC * gzip""" - - LIST_SOFTWARE_DETAILED_MD = """# List of supported software EasyBuild supports 2 different software packages (incl. toolchains, bundles): From 9104b85d9efa6e1c10fbc32d52872f4a2617a770 Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Sun, 26 Nov 2023 22:35:50 +0100 Subject: [PATCH 211/312] Fix verbosity --- easybuild/tools/docs.py | 4 ++-- test/framework/docs.py | 33 ++++++++++++++++++++++++--------- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 74684842dd..40f53f2ef1 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1046,9 +1046,9 @@ def list_software_json(software, detailed=False): x['name'] = key lines.append(json.dumps(x, indent=4) + ",") - if detailed: + if not detailed: break - # remove last comma + # remove last line last comma if len(lines) > 1: lines[-1] = lines[-1][:-1] return '\n'.join(lines) + '\n]' diff --git a/test/framework/docs.py b/test/framework/docs.py index 5894796b05..b52e7e09ee 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -455,15 +455,6 @@ { "name": "GCC" }, -{ - "name": "gzip" -}, -{ - "name": "gzip" -}, -{ - "name": "gzip" -}, { "name": "gzip" } @@ -485,6 +476,30 @@ "version": "1.4", "versionsuffix": "", "name": "gzip" +}, +{ + "toolchain": "system", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.4", + "versionsuffix": "", + "name": "gzip" +}, +{ + "toolchain": "foss/2018a", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.5", + "versionsuffix": "", + "name": "gzip" +}, +{ + "toolchain": "intel/2018a", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.5", + "versionsuffix": "", + "name": "gzip" } ]""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} From 3538e93403c1adf70d9ab12f95537deec634e183 Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Sun, 26 Nov 2023 22:59:42 +0100 Subject: [PATCH 212/312] Add not implemented exceptions for other json cases --- easybuild/tools/docs.py | 43 +++++++++++++++++++++++++++++++++++++++++ test/framework/docs.py | 28 +++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 40f53f2ef1..0c710be74e 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -117,6 +117,11 @@ def avail_cfgfile_constants(go_cfg_constants, output_format=FORMAT_TXT): return generate_doc('avail_cfgfile_constants_%s' % output_format, [go_cfg_constants]) +def avail_cfgfile_constants_json(go_cfg_constants): + """Generate documentation on constants for configuration files in json format""" + raise NotImplementedError("JSON output format not supported for avail_cfgfile_constants_json") + + def avail_cfgfile_constants_txt(go_cfg_constants): """Generate documentation on constants for configuration files in txt format""" doc = [ @@ -186,6 +191,11 @@ def avail_easyconfig_constants(output_format=FORMAT_TXT): return generate_doc('avail_easyconfig_constants_%s' % output_format, []) +def avail_easyconfig_constants_json(): + """Generate easyconfig constant documentation in json format""" + raise NotImplementedError("JSON output format not supported for avail_easyconfig_constants_json") + + def avail_easyconfig_constants_txt(): """Generate easyconfig constant documentation in txt format""" doc = ["Constants that can be used in easyconfigs"] @@ -244,6 +254,11 @@ def avail_easyconfig_licenses(output_format=FORMAT_TXT): return generate_doc('avail_easyconfig_licenses_%s' % output_format, []) +def avail_easyconfig_licenses_json(): + """Generate easyconfig license documentation in json format""" + raise NotImplementedError("JSON output format not supported for avail_easyconfig_licenses_json") + + def avail_easyconfig_licenses_txt(): """Generate easyconfig license documentation in txt format""" doc = ["License constants that can be used in easyconfigs"] @@ -356,6 +371,13 @@ def avail_easyconfig_params_rst(title, grouped_params): return '\n'.join(doc) +def avail_easyconfig_params_json(): + """ + Compose overview of available easyconfig parameters, in json format. + """ + raise NotImplementedError("JSON output format not supported for avail_easyconfig_params_json") + + def avail_easyconfig_params_txt(title, grouped_params): """ Compose overview of available easyconfig parameters, in plain text format. @@ -428,6 +450,11 @@ def avail_easyconfig_templates(output_format=FORMAT_TXT): return generate_doc('avail_easyconfig_templates_%s' % output_format, []) +def avail_easyconfig_templates_json(): + """ Returns template documentation in json text format """ + raise NotImplementedError("JSON output format not supported for avail_easyconfig_templates") + + def avail_easyconfig_templates_txt(): """ Returns template documentation in plain text format """ # This has to reflect the methods/steps used in easyconfig _generate_template_values @@ -642,6 +669,8 @@ def avail_classes_tree(classes, class_names, locations, detailed, format_strings def list_easyblocks(list_easyblocks=SIMPLE, output_format=FORMAT_TXT): + if output_format == FORMAT_JSON: + raise NotImplementedError("JSON output format not supported for list_easyblocks") format_strings = { FORMAT_MD: { 'det_root_templ': "- **%s** (%s%s)", @@ -1202,6 +1231,10 @@ def list_toolchains_txt(tcs): return '\n'.join(doc) +def list_toolchains_json(tcs): + """ Returns overview of all toolchains in json format """ + raise NotImplementedError("JSON output not implemented yet for --list-toolchains") + def avail_toolchain_opts(name, output_format=FORMAT_TXT): """Show list of known options for given toolchain.""" @@ -1256,6 +1289,11 @@ def avail_toolchain_opts_rst(name, tc_dict): return '\n'.join(doc) +def avail_toolchain_opts_json(name, tc_dict): + """ Returns overview of toolchain options in jsonformat """ + raise NotImplementedError("JSON output not implemented yet for --avail-toolchain-opts") + + def avail_toolchain_opts_txt(name, tc_dict): """ Returns overview of toolchain options in txt format """ doc = ["Available options for %s toolchain:" % name] @@ -1281,6 +1319,11 @@ def get_easyblock_classes(package_name): return easyblocks +def gen_easyblocks_overview_json(package_name, path_to_examples, common_params=None, doc_functions=None): + """ + Compose overview of all easyblocks in the given package in json format + """ + raise NotImplementedError("JSON output not implemented yet for gen_easyblocks_overview") def gen_easyblocks_overview_md(package_name, path_to_examples, common_params=None, doc_functions=None): """ diff --git a/test/framework/docs.py b/test/framework/docs.py index b52e7e09ee..a271d63cfc 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -639,6 +639,9 @@ def test_license_docs(self): regex = re.compile(r"^``GPLv3``\s*|The GNU General Public License", re.M) self.assertTrue(regex.search(lic_docs), "%s found in: %s" % (regex.pattern, lic_docs)) + # expect NotImplementedError for JSON output + self.assertRaises(NotImplementedError, avail_easyconfig_licenses, output_format='json') + def test_list_easyblocks(self): """ Tests for list_easyblocks function @@ -667,6 +670,9 @@ def test_list_easyblocks(self): txt = list_easyblocks(list_easyblocks='detailed', output_format='md') self.assertEqual(txt, LIST_EASYBLOCKS_DETAILED_MD % {'topdir': topdir_easyblocks}) + # expect NotImplementedError for JSON output + self.assertRaises(NotImplementedError, list_easyblocks, output_format='json') + def test_list_software(self): """Test list_software* functions.""" build_options = { @@ -791,6 +797,10 @@ def test_list_toolchains(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + # expect NotImplementedError for json output format + with self.assertRaises(NotImplementedError): + list_toolchains(output_format='json') + def test_avail_cfgfile_constants(self): """ Test avail_cfgfile_constants to generate overview of constants that can be used in a configuration file. @@ -835,6 +845,10 @@ def test_avail_cfgfile_constants(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + # expect NotImplementedError for json output format + with self.assertRaises(NotImplementedError): + avail_cfgfile_constants(option_parser.go_cfg_constants, output_format='json') + def test_avail_easyconfig_constants(self): """ Test avail_easyconfig_constants to generate overview of constants that can be used in easyconfig files. @@ -878,6 +892,10 @@ def test_avail_easyconfig_constants(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + # expect NotImplementedError for json output format + with self.assertRaises(NotImplementedError): + avail_easyconfig_constants(output_format='json') + def test_avail_easyconfig_templates(self): """ Test avail_easyconfig_templates to generate overview of templates that can be used in easyconfig files. @@ -928,6 +946,10 @@ def test_avail_easyconfig_templates(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + # expect NotImplementedError for json output format + with self.assertRaises(NotImplementedError): + avail_easyconfig_templates(output_format='json') + def test_avail_toolchain_opts(self): """ Test avail_toolchain_opts to generate overview of supported toolchain options. @@ -1012,6 +1034,12 @@ def test_avail_toolchain_opts(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + # expect NotImplementedError for json output format + with self.assertRaises(NotImplementedError): + avail_toolchain_opts('foss', output_format='json') + with self.assertRaises(NotImplementedError): + avail_toolchain_opts('intel', output_format='json') + def test_mk_table(self): """ Tests for mk_*_table functions. From 5967a66620ab57d3c212dc1e80f99d3e60aa280a Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Sun, 26 Nov 2023 23:01:35 +0100 Subject: [PATCH 213/312] Fix format again --- easybuild/tools/docs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 0c710be74e..344123d0f7 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1231,6 +1231,7 @@ def list_toolchains_txt(tcs): return '\n'.join(doc) + def list_toolchains_json(tcs): """ Returns overview of all toolchains in json format """ raise NotImplementedError("JSON output not implemented yet for --list-toolchains") @@ -1319,12 +1320,14 @@ def get_easyblock_classes(package_name): return easyblocks + def gen_easyblocks_overview_json(package_name, path_to_examples, common_params=None, doc_functions=None): """ Compose overview of all easyblocks in the given package in json format """ raise NotImplementedError("JSON output not implemented yet for gen_easyblocks_overview") + def gen_easyblocks_overview_md(package_name, path_to_examples, common_params=None, doc_functions=None): """ Compose overview of all easyblocks in the given package in MarkDown format From e97eb9b0e0198dca70a6ac89ad217c153706a0d6 Mon Sep 17 00:00:00 2001 From: sam Date: Mon, 27 Nov 2023 17:37:22 +0100 Subject: [PATCH 214/312] extend test to recent toolchain --- test/framework/toolchain.py | 79 +++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index f6243bd3b3..8809278193 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1840,45 +1840,46 @@ def test_old_new_iccifort(self): scalapack_mt_static_libs_fosscuda = "libscalapack.a,libopenblas.a,libgfortran.a,libpthread.a" scalapack_mt_shared_libs_fosscuda = scalapack_mt_static_libs_fosscuda.replace('.a', '.' + shlib_ext) - tc = self.get_toolchain('fosscuda', version='2018a') - tc.prepare() - self.assertEqual(os.environ['BLAS_SHARED_LIBS'], blas_shared_libs_fosscuda) - self.assertEqual(os.environ['BLAS_STATIC_LIBS'], blas_static_libs_fosscuda) - self.assertEqual(os.environ['BLAS_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['BLAS_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) - self.assertEqual(os.environ['LIBBLAS'], libblas_fosscuda) - self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_fosscuda) - - self.assertEqual(os.environ['LAPACK_SHARED_LIBS'], lapack_shared_libs_fosscuda) - self.assertEqual(os.environ['LAPACK_STATIC_LIBS'], lapack_static_libs_fosscuda) - self.assertEqual(os.environ['LAPACK_MT_SHARED_LIBS'], lapack_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['LAPACK_MT_STATIC_LIBS'], lapack_mt_static_libs_fosscuda) - self.assertEqual(os.environ['LIBLAPACK'], liblapack_fosscuda) - self.assertEqual(os.environ['LIBLAPACK_MT'], liblapack_mt_fosscuda) - - self.assertEqual(os.environ['BLAS_LAPACK_SHARED_LIBS'], blas_shared_libs_fosscuda) - self.assertEqual(os.environ['BLAS_LAPACK_STATIC_LIBS'], blas_static_libs_fosscuda) - self.assertEqual(os.environ['BLAS_LAPACK_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['BLAS_LAPACK_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) - - self.assertEqual(os.environ['FFT_SHARED_LIBS'], fft_shared_libs_fosscuda) - self.assertEqual(os.environ['FFT_STATIC_LIBS'], fft_static_libs_fosscuda) - self.assertEqual(os.environ['FFT_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['FFT_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) - self.assertEqual(os.environ['FFTW_SHARED_LIBS'], fft_shared_libs_fosscuda) - self.assertEqual(os.environ['FFTW_STATIC_LIBS'], fft_static_libs_fosscuda) - self.assertEqual(os.environ['FFTW_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['FFTW_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) - self.assertEqual(os.environ['LIBFFT'], libfft_fosscuda) - self.assertEqual(os.environ['LIBFFT_MT'], libfft_mt_fosscuda) - - self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_fosscuda) - self.assertEqual(os.environ['LIBSCALAPACK_MT'], libscalack_mt_fosscuda) - self.assertEqual(os.environ['SCALAPACK_SHARED_LIBS'], scalapack_shared_libs_fosscuda) - self.assertEqual(os.environ['SCALAPACK_STATIC_LIBS'], scalapack_static_libs_fosscuda) - self.assertEqual(os.environ['SCALAPACK_MT_SHARED_LIBS'], scalapack_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['SCALAPACK_MT_STATIC_LIBS'], scalapack_mt_static_libs_fosscuda) - self.modtool.purge() + for tc in [('fosscuda', '2018a'), ('foss', '2023a')]: + tc = self.get_toolchain(tc[0], version=tc[1]) + tc.prepare() + self.assertEqual(os.environ['BLAS_SHARED_LIBS'], blas_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_STATIC_LIBS'], blas_static_libs_fosscuda) + self.assertEqual(os.environ['BLAS_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBBLAS'], libblas_fosscuda) + self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_fosscuda) + + self.assertEqual(os.environ['LAPACK_SHARED_LIBS'], lapack_shared_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_STATIC_LIBS'], lapack_static_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_MT_SHARED_LIBS'], lapack_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_MT_STATIC_LIBS'], lapack_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBLAPACK'], liblapack_fosscuda) + self.assertEqual(os.environ['LIBLAPACK_MT'], liblapack_mt_fosscuda) + + self.assertEqual(os.environ['BLAS_LAPACK_SHARED_LIBS'], blas_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_STATIC_LIBS'], blas_static_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) + + self.assertEqual(os.environ['FFT_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFT_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBFFT'], libfft_fosscuda) + self.assertEqual(os.environ['LIBFFT_MT'], libfft_mt_fosscuda) + + self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_fosscuda) + self.assertEqual(os.environ['LIBSCALAPACK_MT'], libscalack_mt_fosscuda) + self.assertEqual(os.environ['SCALAPACK_SHARED_LIBS'], scalapack_shared_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_STATIC_LIBS'], scalapack_static_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_MT_SHARED_LIBS'], scalapack_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_MT_STATIC_LIBS'], scalapack_mt_static_libs_fosscuda) + self.modtool.purge() tc = self.get_toolchain('intel', version='2018a') tc.prepare() From e11b9bd7ac62ebd3508dd6240e3f06b68f1a9c62 Mon Sep 17 00:00:00 2001 From: sam Date: Mon, 27 Nov 2023 17:47:53 +0100 Subject: [PATCH 215/312] fix --- test/framework/toolchain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 8809278193..bee0eec8a9 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1840,8 +1840,8 @@ def test_old_new_iccifort(self): scalapack_mt_static_libs_fosscuda = "libscalapack.a,libopenblas.a,libgfortran.a,libpthread.a" scalapack_mt_shared_libs_fosscuda = scalapack_mt_static_libs_fosscuda.replace('.a', '.' + shlib_ext) - for tc in [('fosscuda', '2018a'), ('foss', '2023a')]: - tc = self.get_toolchain(tc[0], version=tc[1]) + for toolc in [('fosscuda', '2018a'), ('foss', '2023a')]: + tc = self.get_toolchain(toolc[0], version=toolc[1]) tc.prepare() self.assertEqual(os.environ['BLAS_SHARED_LIBS'], blas_shared_libs_fosscuda) self.assertEqual(os.environ['BLAS_STATIC_LIBS'], blas_static_libs_fosscuda) From 7e6993fafdab1c78391bc8c494018cf92848cbad Mon Sep 17 00:00:00 2001 From: sam Date: Mon, 27 Nov 2023 18:40:46 +0100 Subject: [PATCH 216/312] add foss/2023a module for testing --- test/framework/modules/foss/2023a | 45 +++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 test/framework/modules/foss/2023a diff --git a/test/framework/modules/foss/2023a b/test/framework/modules/foss/2023a new file mode 100644 index 0000000000..05540a45f5 --- /dev/null +++ b/test/framework/modules/foss/2023a @@ -0,0 +1,45 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { GCC based compiler toolchain including + OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. - Homepage: (none) +} +} + +module-whatis {GCC based compiler toolchain including + OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. - Homepage: (none)} + +set root /prefix/software/foss/2023a + +conflict foss + +if { ![is-loaded GCC/12.3.0] } { + module load GCC/12.3.0 +} + +if { ![is-loaded OpenMPI/4.1.5-GCC-12.3.0] } { + module load OpenMPI/4.1.5-GCC-12.3.0 +} + +if { ![is-loaded FlexiBLAS/3.3.1-GCC-12.3.0] } { + module load FlexiBLAS/3.3.1-GCC-12.3.0 +} + +if { ![is-loaded FFTW/3.3.10-GCC-12.3.0] } { + module load FFTW/3.3.10-GCC-12.3.0 +} + +if { ![is-loaded FFTW.MPI/3.3.10-gompi-2023a] } { + module load FFTW.MPI/3.3.10-gompi-2023a +} + +if { ![is-loaded ScaLAPACK/2.2.0-gompi-2023a-fb] } { + module load ScaLAPACK/2.2.0-gompi-2023a-fb +} + +setenv EBROOTFOSS "$root" +setenv EBVERSIONFOSS "2023a" +setenv EBDEVELFOSS "$root/easybuild/foss-2023a-easybuild-devel" + + +# built with EasyBuild version 1.4.0dev From 737d77895d4003c277989811da6a04aee4e91208 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Tue, 28 Nov 2023 10:43:11 +0000 Subject: [PATCH 217/312] resolve templated values in name in _make_extension_list --- easybuild/framework/easyblock.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 3ac26eb476..7df6d9abae 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1739,9 +1739,15 @@ def _make_extension_list(self): Each entry should be a (name, version) tuple or just (name, ) if no version exists """ - # We need only name and version, so don't resolve templates # Each extension in exts_list is either a string or a list/tuple with name, version as first entries - return [(ext, ) if isinstance(ext, string_type) else ext[:2] for ext in self.cfg.get_ref('exts_list')] + # As name can be a templated value we must resolve templates + exts_list = [] + for ext in self.cfg.get_ref('exts_list'): + if isinstance(ext, string_type): + exts_list.append((resolve_template(ext, self.cfg.template_values), )) + else: + exts_list.append((resolve_template(ext[0], self.cfg.template_values), ext[1])) + return exts_list def make_extension_string(self, name_version_sep='-', ext_sep=', ', sort=True): """ From 873a001bb9bf98512ef218f9f103aa262cc218a1 Mon Sep 17 00:00:00 2001 From: sam Date: Sat, 2 Dec 2023 20:21:44 +0100 Subject: [PATCH 218/312] add deps --- .../modules/FFTW.MPI/3.3.10-gompi-2023a | 42 +++++++++++++++++ test/framework/modules/FFTW/3.3.10-GCC-12.3.0 | 43 +++++++++++++++++ .../modules/FlexiBLAS/3.3.1-GCC-12.3.0 | 47 +++++++++++++++++++ test/framework/modules/GCC/12.3.0 | 38 +++++++++++++++ test/framework/modules/GCCcore/12.3.0 | 37 +++++++++++++++ .../modules/OpenBLAS/0.3.23-GCC-12.3.0 | 38 +++++++++++++++ .../modules/ScaLAPACK/2.2.0-gompi-2023a-fb | 43 +++++++++++++++++ .../modules/binutils/2.40-GCCcore-12.3.0 | 44 +++++++++++++++++ test/framework/modules/foss/2023a | 41 +++++++++------- .../modules/zlib/1.2.13-GCCcore-12.3.0 | 44 +++++++++++++++++ 10 files changed, 401 insertions(+), 16 deletions(-) create mode 100644 test/framework/modules/FFTW.MPI/3.3.10-gompi-2023a create mode 100644 test/framework/modules/FFTW/3.3.10-GCC-12.3.0 create mode 100644 test/framework/modules/FlexiBLAS/3.3.1-GCC-12.3.0 create mode 100644 test/framework/modules/GCC/12.3.0 create mode 100644 test/framework/modules/GCCcore/12.3.0 create mode 100644 test/framework/modules/OpenBLAS/0.3.23-GCC-12.3.0 create mode 100644 test/framework/modules/ScaLAPACK/2.2.0-gompi-2023a-fb create mode 100644 test/framework/modules/binutils/2.40-GCCcore-12.3.0 create mode 100644 test/framework/modules/zlib/1.2.13-GCCcore-12.3.0 diff --git a/test/framework/modules/FFTW.MPI/3.3.10-gompi-2023a b/test/framework/modules/FFTW.MPI/3.3.10-gompi-2023a new file mode 100644 index 0000000000..e35c12d348 --- /dev/null +++ b/test/framework/modules/FFTW.MPI/3.3.10-gompi-2023a @@ -0,0 +1,42 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data. + + +More information +================ + - Homepage: https://www.fftw.org + } +} + +module-whatis {Description: FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data.} +module-whatis {Homepage: https://www.fftw.org} +module-whatis {URL: https://www.fftw.org} + +set root /prefix/software/FFTW.MPI/3.3.10-gompi-2023a + +conflict FFTW.MPI + +if { ![ is-loaded gompi/2023a ] } { + module load gompi/2023a +} + +if { ![ is-loaded FFTW/3.3.10-GCC-12.3.0 ] } { + module load FFTW/3.3.10-GCC-12.3.0 +} + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +setenv EBROOTFFTWMPI "$root" +setenv EBVERSIONFFTWMPI "3.3.10" +setenv EBDEVELFFTWMPI "$root/easybuild/FFTW.MPI-3.3.10-gompi-2023a-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/FFTW/3.3.10-GCC-12.3.0 b/test/framework/modules/FFTW/3.3.10-GCC-12.3.0 new file mode 100644 index 0000000000..2ae82a0193 --- /dev/null +++ b/test/framework/modules/FFTW/3.3.10-GCC-12.3.0 @@ -0,0 +1,43 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data. + + +More information +================ + - Homepage: https://www.fftw.org + } +} + +module-whatis {Description: FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data.} +module-whatis {Homepage: https://www.fftw.org} +module-whatis {URL: https://www.fftw.org} + +set root /prefix/software/FFTW/3.3.10-GCC-12.3.0 + +conflict FFTW + +if { ![ is-loaded GCC/12.3.0 ] } { + module load GCC/12.3.0 +} + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig +prepend-path XDG_DATA_DIRS $root/share + +setenv EBROOTFFTW "$root" +setenv EBVERSIONFFTW "3.3.10" +setenv EBDEVELFFTW "$root/easybuild/FFTW-3.3.10-GCC-12.3.0-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/FlexiBLAS/3.3.1-GCC-12.3.0 b/test/framework/modules/FlexiBLAS/3.3.1-GCC-12.3.0 new file mode 100644 index 0000000000..00f02d0460 --- /dev/null +++ b/test/framework/modules/FlexiBLAS/3.3.1-GCC-12.3.0 @@ -0,0 +1,47 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +FlexiBLAS is a wrapper library that enables the exchange of the BLAS and LAPACK implementation +used by a program without recompiling or relinking it. + + +More information +================ + - Homepage: https://gitlab.mpi-magdeburg.mpg.de/software/flexiblas-release + } +} + +module-whatis {Description: FlexiBLAS is a wrapper library that enables the exchange of the BLAS and LAPACK implementation +used by a program without recompiling or relinking it.} +module-whatis {Homepage: https://gitlab.mpi-magdeburg.mpg.de/software/flexiblas-release} +module-whatis {URL: https://gitlab.mpi-magdeburg.mpg.de/software/flexiblas-release} + +set root /prefix/software/FlexiBLAS/3.3.1-GCC-12.3.0 + +conflict FlexiBLAS + +if { ![ is-loaded GCC/12.3.0 ] } { + module load GCC/12.3.0 +} + +if { ![ is-loaded OpenBLAS/0.3.23-GCC-12.3.0 ] } { + module load OpenBLAS/0.3.23-GCC-12.3.0 +} + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig +prepend-path XDG_DATA_DIRS $root/share + +setenv EBROOTFLEXIBLAS "$root" +setenv EBVERSIONFLEXIBLAS "3.3.1" +setenv EBDEVELFLEXIBLAS "$root/easybuild/FlexiBLAS-3.3.1-GCC-12.3.0-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/GCC/12.3.0 b/test/framework/modules/GCC/12.3.0 new file mode 100644 index 0000000000..5a21e5af1c --- /dev/null +++ b/test/framework/modules/GCC/12.3.0 @@ -0,0 +1,38 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...). + + +More information +================ + - Homepage: https://gcc.gnu.org/ + } +} + +module-whatis {Description: The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...).} +module-whatis {Homepage: https://gcc.gnu.org/} +module-whatis {URL: https://gcc.gnu.org/} + +set root /prefix/software/GCC/12.3.0 + +conflict GCC + +if { ![ is-loaded GCCcore/12.3.0 ] } { + module load GCCcore/12.3.0 +} + +if { ![ is-loaded binutils/2.40-GCCcore-12.3.0 ] } { + module load binutils/2.40-GCCcore-12.3.0 +} + +setenv EBROOTGCC "$root" +setenv EBVERSIONGCC "12.3.0" +setenv EBDEVELGCC "$root/easybuild/GCC-12.3.0-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/GCCcore/12.3.0 b/test/framework/modules/GCCcore/12.3.0 new file mode 100644 index 0000000000..745b111007 --- /dev/null +++ b/test/framework/modules/GCCcore/12.3.0 @@ -0,0 +1,37 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...). + + +More information +================ + - Homepage: https://gcc.gnu.org/ + } +} + +module-whatis {Description: The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...).} +module-whatis {Homepage: https://gcc.gnu.org/} +module-whatis {URL: https://gcc.gnu.org/} + +set root /prefix/software/GCCcore/12.3.0 + +conflict GCCcore + +prepend-path CMAKE_LIBRARY_PATH $root/lib64 +prepend-path CMAKE_PREFIX_PATH $root +prepend-path LD_LIBRARY_PATH $root/lib64 +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path XDG_DATA_DIRS $root/share + +setenv EBROOTGCCCORE "$root" +setenv EBVERSIONGCCCORE "12.3.0" +setenv EBDEVELGCCCORE "$root/easybuild/GCCcore-12.3.0-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/OpenBLAS/0.3.23-GCC-12.3.0 b/test/framework/modules/OpenBLAS/0.3.23-GCC-12.3.0 new file mode 100644 index 0000000000..89a72a65bc --- /dev/null +++ b/test/framework/modules/OpenBLAS/0.3.23-GCC-12.3.0 @@ -0,0 +1,38 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version. + + +More information +================ + - Homepage: http://www.openblas.net/ + } +} + +module-whatis {Description: OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version.} +module-whatis {Homepage: http://www.openblas.net/} +module-whatis {URL: http://www.openblas.net/} + +set root /prefix/software/OpenBLAS/0.3.23-GCC-12.3.0 + +conflict OpenBLAS + +if { ![ is-loaded GCC/12.3.0 ] } { + module load GCC/12.3.0 +} + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig + +setenv EBROOTOPENBLAS "$root" +setenv EBVERSIONOPENBLAS "0.3.23" +setenv EBDEVELOPENBLAS "$root/easybuild/OpenBLAS-0.3.23-GCC-12.3.0-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/ScaLAPACK/2.2.0-gompi-2023a-fb b/test/framework/modules/ScaLAPACK/2.2.0-gompi-2023a-fb new file mode 100644 index 0000000000..7118eebf4f --- /dev/null +++ b/test/framework/modules/ScaLAPACK/2.2.0-gompi-2023a-fb @@ -0,0 +1,43 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +The ScaLAPACK (or Scalable LAPACK) library includes a subset of LAPACK routines + redesigned for distributed memory MIMD parallel computers. + + +More information +================ + - Homepage: https://www.netlib.org/scalapack/ + } +} + +module-whatis {Description: The ScaLAPACK (or Scalable LAPACK) library includes a subset of LAPACK routines + redesigned for distributed memory MIMD parallel computers.} +module-whatis {Homepage: https://www.netlib.org/scalapack/} +module-whatis {URL: https://www.netlib.org/scalapack/} + +set root /prefix/software/ScaLAPACK/2.2.0-gompi-2023a-fb + +conflict ScaLAPACK + +if { ![ is-loaded gompi/2023a ] } { + module load gompi/2023a +} + +if { ![ is-loaded FlexiBLAS/3.3.1-GCC-12.3.0 ] } { + module load FlexiBLAS/3.3.1-GCC-12.3.0 +} + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig + +setenv EBROOTSCALAPACK "$root" +setenv EBVERSIONSCALAPACK "2.2.0" +setenv EBDEVELSCALAPACK "$root/easybuild/ScaLAPACK-2.2.0-gompi-2023a-fb-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/binutils/2.40-GCCcore-12.3.0 b/test/framework/modules/binutils/2.40-GCCcore-12.3.0 new file mode 100644 index 0000000000..7975adcb60 --- /dev/null +++ b/test/framework/modules/binutils/2.40-GCCcore-12.3.0 @@ -0,0 +1,44 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +binutils: GNU binary utilities + + +More information +================ + - Homepage: https://directory.fsf.org/project/binutils/ + } +} + +module-whatis {Description: binutils: GNU binary utilities} +module-whatis {Homepage: https://directory.fsf.org/project/binutils/} +module-whatis {URL: https://directory.fsf.org/project/binutils/} + +set root /prefix/software/binutils/2.40-GCCcore-12.3.0 + +conflict binutils + +if { ![ is-loaded GCCcore/12.3.0 ] } { + module load GCCcore/12.3.0 +} + +if { ![ is-loaded zlib/1.2.13-GCCcore-12.3.0 ] } { + module load zlib/1.2.13-GCCcore-12.3.0 +} + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path XDG_DATA_DIRS $root/share + +setenv EBROOTBINUTILS "$root" +setenv EBVERSIONBINUTILS "2.40" +setenv EBDEVELBINUTILS "$root/easybuild/binutils-2.40-GCCcore-12.3.0-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/foss/2023a b/test/framework/modules/foss/2023a index 05540a45f5..448c5a77be 100644 --- a/test/framework/modules/foss/2023a +++ b/test/framework/modules/foss/2023a @@ -1,39 +1,49 @@ #%Module - proc ModulesHelp { } { - puts stderr { GCC based compiler toolchain including - OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. - Homepage: (none) -} + puts stderr { + +Description +=========== +GNU Compiler Collection (GCC) based compiler toolchain, including + OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. + + +More information +================ + - Homepage: https://easybuild.readthedocs.io/en/master/Common-toolchains.html#foss-toolchain + } } -module-whatis {GCC based compiler toolchain including - OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. - Homepage: (none)} +module-whatis {Description: GNU Compiler Collection (GCC) based compiler toolchain, including + OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK.} +module-whatis {Homepage: https://easybuild.readthedocs.io/en/master/Common-toolchains.html#foss-toolchain} +module-whatis {URL: https://easybuild.readthedocs.io/en/master/Common-toolchains.html#foss-toolchain} -set root /prefix/software/foss/2023a +set root /prefix/software/foss/2023a -conflict foss +conflict foss -if { ![is-loaded GCC/12.3.0] } { +if { ![ is-loaded GCC/12.3.0 ] } { module load GCC/12.3.0 } -if { ![is-loaded OpenMPI/4.1.5-GCC-12.3.0] } { +if { ![ is-loaded OpenMPI/4.1.5-GCC-12.3.0 ] } { module load OpenMPI/4.1.5-GCC-12.3.0 } -if { ![is-loaded FlexiBLAS/3.3.1-GCC-12.3.0] } { +if { ![ is-loaded FlexiBLAS/3.3.1-GCC-12.3.0 ] } { module load FlexiBLAS/3.3.1-GCC-12.3.0 } -if { ![is-loaded FFTW/3.3.10-GCC-12.3.0] } { +if { ![ is-loaded FFTW/3.3.10-GCC-12.3.0 ] } { module load FFTW/3.3.10-GCC-12.3.0 } -if { ![is-loaded FFTW.MPI/3.3.10-gompi-2023a] } { +if { ![ is-loaded FFTW.MPI/3.3.10-gompi-2023a ] } { module load FFTW.MPI/3.3.10-gompi-2023a } -if { ![is-loaded ScaLAPACK/2.2.0-gompi-2023a-fb] } { +if { ![ is-loaded ScaLAPACK/2.2.0-gompi-2023a-fb ] } { module load ScaLAPACK/2.2.0-gompi-2023a-fb } @@ -41,5 +51,4 @@ setenv EBROOTFOSS "$root" setenv EBVERSIONFOSS "2023a" setenv EBDEVELFOSS "$root/easybuild/foss-2023a-easybuild-devel" - -# built with EasyBuild version 1.4.0dev +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/zlib/1.2.13-GCCcore-12.3.0 b/test/framework/modules/zlib/1.2.13-GCCcore-12.3.0 new file mode 100644 index 0000000000..a82945bd0c --- /dev/null +++ b/test/framework/modules/zlib/1.2.13-GCCcore-12.3.0 @@ -0,0 +1,44 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +zlib is designed to be a free, general-purpose, legally unencumbered -- that is, + not covered by any patents -- lossless data-compression library for use on virtually any + computer hardware and operating system. + + +More information +================ + - Homepage: https://www.zlib.net/ + } +} + +module-whatis {Description: zlib is designed to be a free, general-purpose, legally unencumbered -- that is, + not covered by any patents -- lossless data-compression library for use on virtually any + computer hardware and operating system.} +module-whatis {Homepage: https://www.zlib.net/} +module-whatis {URL: https://www.zlib.net/} + +set root /prefix/software/zlib/1.2.13-GCCcore-12.3.0 + +conflict zlib + +if { ![ is-loaded GCCcore/12.3.0 ] } { + module load GCCcore/12.3.0 +} + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig +prepend-path XDG_DATA_DIRS $root/share + +setenv EBROOTZLIB "$root" +setenv EBVERSIONZLIB "1.2.13" +setenv EBDEVELZLIB "$root/easybuild/zlib-1.2.13-GCCcore-12.3.0-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 From edde7796fde519b7b75058d52295729fdfd8a4e0 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 5 Dec 2023 10:10:07 +0100 Subject: [PATCH 219/312] use source toolchain version when passing only `--try-toolchain` `process_software_build_specs` ialways adds a `toolchain` key even when only the name of the toolchain is available. This makes the code in `tweak` ignore the code path in using `toolchain_name` and/or `toolchain_version` with the fallback to using the values of the source toolchain. This in turn leads to failures further down when a value of `None` is used where an actual version is expected. Fix by only adding the `toolchain` key when we have both the name and version and hence a complete, valid toolchain spec. --- easybuild/tools/options.py | 6 ++-- .../test_ecs/i/iimpi/iimpi-2018a.eb | 20 +++++++++++ .../test_ecs/i/intel/intel-2018a.eb | 15 +++++---- .../test_ecs/t/toy/toy-0.0-foss-2018a.eb | 33 +++++++++++++++++++ test/framework/options.py | 12 +++++++ 5 files changed, 78 insertions(+), 8 deletions(-) create mode 100644 test/framework/easyconfigs/test_ecs/i/iimpi/iimpi-2018a.eb create mode 100644 test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-foss-2018a.eb diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index e69d96b2f6..eeba800c09 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1735,11 +1735,13 @@ def process_software_build_specs(options): }) # provide both toolchain and toolchain_name/toolchain_version keys - if 'toolchain_name' in build_specs: + try: build_specs['toolchain'] = { 'name': build_specs['toolchain_name'], - 'version': build_specs.get('toolchain_version', None), + 'version': build_specs['toolchain_version'], } + except KeyError: + pass # Don't set toolchain key if we don't have both keys # process --amend and --try-amend if options.amend or options.try_amend: diff --git a/test/framework/easyconfigs/test_ecs/i/iimpi/iimpi-2018a.eb b/test/framework/easyconfigs/test_ecs/i/iimpi/iimpi-2018a.eb new file mode 100644 index 0000000000..008b76a1b4 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/i/iimpi/iimpi-2018a.eb @@ -0,0 +1,20 @@ +# This is an easyconfig file for EasyBuild, see http://easybuilders.github.io/easybuild +easyblock = 'Toolchain' + +name = 'iimpi' +version = '2018a' + +homepage = 'https://software.intel.com/parallel-studio-xe' +description = """Intel C/C++ and Fortran compilers, alongside Intel MPI.""" + +toolchain = SYSTEM + +local_compver = '2016.1.150' +local_suff = '-GCC-4.9.3-2.25' +dependencies = [ + ('icc', local_compver, local_suff), + ('ifort', local_compver, local_suff), + ('impi', '5.1.2.150', '', ('iccifort', '%s%s' % (local_compver, local_suff))), +] + +moduleclass = 'toolchain' diff --git a/test/framework/easyconfigs/test_ecs/i/intel/intel-2018a.eb b/test/framework/easyconfigs/test_ecs/i/intel/intel-2018a.eb index af32ecb585..8fdf23171b 100644 --- a/test/framework/easyconfigs/test_ecs/i/intel/intel-2018a.eb +++ b/test/framework/easyconfigs/test_ecs/i/intel/intel-2018a.eb @@ -8,14 +8,17 @@ description = """Intel Cluster Toolkit Compiler Edition provides Intel C/C++ and toolchain = SYSTEM +local_compver = '2016.1.150' +local_gccver = '4.9.3' +local_binutilsver = '2.25' +local_gccsuff = '-GCC-%s-%s' % (local_gccver, local_binutilsver) # fake intel toolchain easyconfig, no dependencies (good enough for testing) local_fake_dependencies = [ - ('GCCcore', '6.4.0'), - ('binutils', '2.28', '-GCCcore-6.4.0'), - ('icc', '2018.1.163', '-GCCcore-6.4.0'), - ('ifort', '2018.1.163', '-GCCcore-6.4.0'), - ('impi', '2018.1.163', '', ('iccifort', '2018.1.163-GCCcore-6.4.0')), - ('imkl', '2018.1.163', '', ('iimpi', version)), + ('GCCcore', local_gccver), + ('binutils', local_binutilsver, '-GCCcore-%s' % local_gccver), + ('icc', local_compver, local_gccsuff), + ('ifort', local_compver, local_gccsuff), + ('impi', '5.1.2.150', '', ('iccifort', '%s%s' % (local_compver, local_gccsuff))), ] moduleclass = 'toolchain' diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-foss-2018a.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-foss-2018a.eb new file mode 100644 index 0000000000..5f95d5d108 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-foss-2018a.eb @@ -0,0 +1,33 @@ +name = 'toy' +version = '0.0' + +homepage = 'https://easybuilders.github.io/easybuild' +description = "Toy C program, 100% toy." + +toolchain = {'name': 'foss', 'version': '2018a'} +toolchainopts = {'pic': True, 'opt': True, 'optarch': True} + +sources = [SOURCE_TAR_GZ] +checksums = [[ + 'be662daa971a640e40be5c804d9d7d10', # default (MD5) + '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256) + ('adler32', '0x998410035'), + ('crc32', '0x1553842328'), + ('md5', 'be662daa971a640e40be5c804d9d7d10'), + ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'), + ('size', 273), +]] +patches = [ + 'toy-0.0_fix-silly-typo-in-printf-statement.patch', + ('toy-extra.txt', 'toy-0.0'), +] + +sanity_check_paths = { + 'files': [('bin/yot', 'bin/toy')], + 'dirs': ['bin'], +} + +postinstallcmds = ["echo TOY > %(installdir)s/README"] + +moduleclass = 'tools' +# trailing comment, leave this here, it may trigger bugs with extract_comments() diff --git a/test/framework/options.py b/test/framework/options.py index 51b67e39e5..050c123e0d 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2420,6 +2420,18 @@ def test_try(self): allargs = args + ['--software-version=1.2.3', '--toolchain=gompi,2018a'] self.assertErrorRegex(EasyBuildError, "version .* not available", self.eb_main, allargs, raise_error=True) + # Try changing only name or version of toolchain + args.pop(0) # Remove EC filename + test_cases = [ + (['toy-0.0-gompi-2018a.eb', '--try-toolchain-name=intel'], 'toy/0.0-iimpi-2018a'), + (['toy-0.0-foss-2018a.eb', '--try-toolchain-name=intel'], 'toy/0.0-intel-2018a'), + (['toy-0.0-gompi-2018a.eb', '--try-toolchain-version=2018b'], 'toy/0.0-gompi-2018b'), + ] + for extra_args, mod in test_cases: + outtxt = self.eb_main(args + extra_args, verbose=True, raise_error=True) + mod_regex = re.compile(r"\(module: %s\)$" % mod, re.M) + self.assertTrue(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) + def test_try_with_copy(self): """Test whether --try options are taken into account.""" ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') From 9613d9f10bee1408b473637d029535327db875c8 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 5 Dec 2023 11:13:43 +0100 Subject: [PATCH 220/312] Avoid interferring with other tests --- .../test_ecs/t/toy/toy-0.0-foss-2018a.eb | 33 ------------------- test/framework/filetools.py | 2 +- test/framework/options.py | 6 +++- 3 files changed, 6 insertions(+), 35 deletions(-) delete mode 100644 test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-foss-2018a.eb diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-foss-2018a.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-foss-2018a.eb deleted file mode 100644 index 5f95d5d108..0000000000 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-foss-2018a.eb +++ /dev/null @@ -1,33 +0,0 @@ -name = 'toy' -version = '0.0' - -homepage = 'https://easybuilders.github.io/easybuild' -description = "Toy C program, 100% toy." - -toolchain = {'name': 'foss', 'version': '2018a'} -toolchainopts = {'pic': True, 'opt': True, 'optarch': True} - -sources = [SOURCE_TAR_GZ] -checksums = [[ - 'be662daa971a640e40be5c804d9d7d10', # default (MD5) - '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256) - ('adler32', '0x998410035'), - ('crc32', '0x1553842328'), - ('md5', 'be662daa971a640e40be5c804d9d7d10'), - ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'), - ('size', 273), -]] -patches = [ - 'toy-0.0_fix-silly-typo-in-printf-statement.patch', - ('toy-extra.txt', 'toy-0.0'), -] - -sanity_check_paths = { - 'files': [('bin/yot', 'bin/toy')], - 'dirs': ['bin'], -} - -postinstallcmds = ["echo TOY > %(installdir)s/README"] - -moduleclass = 'tools' -# trailing comment, leave this here, it may trigger bugs with extract_comments() diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 400a05c1b3..3b879bef85 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2401,7 +2401,7 @@ def test_index_functions(self): # test with specified path with and without trailing '/'s for path in [test_ecs, test_ecs + '/', test_ecs + '//']: index = ft.create_index(path) - self.assertEqual(len(index), 91) + self.assertEqual(len(index), 92) expected = [ os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), diff --git a/test/framework/options.py b/test/framework/options.py index 050c123e0d..e6be49ab4f 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2422,9 +2422,13 @@ def test_try(self): # Try changing only name or version of toolchain args.pop(0) # Remove EC filename + foss_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-foss-2018a.eb') + copy_file(os.path.join(ecs_path, 't', 'toy', 'toy-0.0-gompi-2018a.eb'), foss_toy_ec) + write_file(foss_toy_ec, "toolchain['name'] = 'foss'", append=True) + test_cases = [ (['toy-0.0-gompi-2018a.eb', '--try-toolchain-name=intel'], 'toy/0.0-iimpi-2018a'), - (['toy-0.0-foss-2018a.eb', '--try-toolchain-name=intel'], 'toy/0.0-intel-2018a'), + ([foss_toy_ec, '--try-toolchain-name=intel'], 'toy/0.0-intel-2018a'), (['toy-0.0-gompi-2018a.eb', '--try-toolchain-version=2018b'], 'toy/0.0-gompi-2018b'), ] for extra_args, mod in test_cases: From aaeaa62bb72b522d0769105b750ba62a5679547e Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Wed, 6 Dec 2023 11:53:37 +0100 Subject: [PATCH 221/312] Add trailing spaces --- test/framework/docs.py | 50 +++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/test/framework/docs.py b/test/framework/docs.py index a271d63cfc..4f878fc728 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -462,43 +462,43 @@ LIST_SOFTWARE_DETAILED_JSON = """[ { - "toolchain": "system", - "description": "%(gcc_descr)s", - "homepage": "http://gcc.gnu.org/", - "version": "4.6.3", - "versionsuffix": "", + "toolchain": "system", + "description": "%(gcc_descr)s", + "homepage": "http://gcc.gnu.org/", + "version": "4.6.3", + "versionsuffix": "", "name": "GCC" }, { - "toolchain": "GCC/4.6.3", - "description": "%(gzip_descr)s", - "homepage": "http://www.gzip.org/", - "version": "1.4", - "versionsuffix": "", + "toolchain": "GCC/4.6.3", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.4", + "versionsuffix": "", "name": "gzip" }, { - "toolchain": "system", - "description": "%(gzip_descr)s", - "homepage": "http://www.gzip.org/", - "version": "1.4", - "versionsuffix": "", + "toolchain": "system", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.4", + "versionsuffix": "", "name": "gzip" }, { - "toolchain": "foss/2018a", - "description": "%(gzip_descr)s", - "homepage": "http://www.gzip.org/", - "version": "1.5", - "versionsuffix": "", + "toolchain": "foss/2018a", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.5", + "versionsuffix": "", "name": "gzip" }, { - "toolchain": "intel/2018a", - "description": "%(gzip_descr)s", - "homepage": "http://www.gzip.org/", - "version": "1.5", - "versionsuffix": "", + "toolchain": "intel/2018a", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.5", + "versionsuffix": "", "name": "gzip" } ]""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} From d2ccb1aa3364bf251b3a50cb973b98184e3977ed Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Wed, 6 Dec 2023 12:22:00 +0100 Subject: [PATCH 222/312] Enforce no space after comma from json.dumps (for older pythons) --- easybuild/tools/docs.py | 2 +- test/framework/docs.py | 50 ++++++++++++++++++++--------------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 344123d0f7..1bbb4c2d59 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1074,7 +1074,7 @@ def list_software_json(software, detailed=False): x = {} x['name'] = key - lines.append(json.dumps(x, indent=4) + ",") + lines.append(json.dumps(x, indent=4, separators=(',', ': ')) + ",") if not detailed: break # remove last line last comma diff --git a/test/framework/docs.py b/test/framework/docs.py index 4f878fc728..a271d63cfc 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -462,43 +462,43 @@ LIST_SOFTWARE_DETAILED_JSON = """[ { - "toolchain": "system", - "description": "%(gcc_descr)s", - "homepage": "http://gcc.gnu.org/", - "version": "4.6.3", - "versionsuffix": "", + "toolchain": "system", + "description": "%(gcc_descr)s", + "homepage": "http://gcc.gnu.org/", + "version": "4.6.3", + "versionsuffix": "", "name": "GCC" }, { - "toolchain": "GCC/4.6.3", - "description": "%(gzip_descr)s", - "homepage": "http://www.gzip.org/", - "version": "1.4", - "versionsuffix": "", + "toolchain": "GCC/4.6.3", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.4", + "versionsuffix": "", "name": "gzip" }, { - "toolchain": "system", - "description": "%(gzip_descr)s", - "homepage": "http://www.gzip.org/", - "version": "1.4", - "versionsuffix": "", + "toolchain": "system", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.4", + "versionsuffix": "", "name": "gzip" }, { - "toolchain": "foss/2018a", - "description": "%(gzip_descr)s", - "homepage": "http://www.gzip.org/", - "version": "1.5", - "versionsuffix": "", + "toolchain": "foss/2018a", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.5", + "versionsuffix": "", "name": "gzip" }, { - "toolchain": "intel/2018a", - "description": "%(gzip_descr)s", - "homepage": "http://www.gzip.org/", - "version": "1.5", - "versionsuffix": "", + "toolchain": "intel/2018a", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.5", + "versionsuffix": "", "name": "gzip" } ]""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} From 5423a6fcedd1617ef5cd30883a8c0485abb8b62b Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Wed, 6 Dec 2023 13:38:52 +0100 Subject: [PATCH 223/312] Sort JSON keys for older python versions --- easybuild/tools/docs.py | 2 +- test/framework/docs.py | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 1bbb4c2d59..43f6038b26 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1074,7 +1074,7 @@ def list_software_json(software, detailed=False): x = {} x['name'] = key - lines.append(json.dumps(x, indent=4, separators=(',', ': ')) + ",") + lines.append(json.dumps(x, indent=4, sort_keys=True, separators=(',', ': ')) + ",") if not detailed: break # remove last line last comma diff --git a/test/framework/docs.py b/test/framework/docs.py index a271d63cfc..70280892e4 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -462,44 +462,44 @@ LIST_SOFTWARE_DETAILED_JSON = """[ { - "toolchain": "system", "description": "%(gcc_descr)s", "homepage": "http://gcc.gnu.org/", + "name": "GCC", + "toolchain": "system", "version": "4.6.3", - "versionsuffix": "", - "name": "GCC" + "versionsuffix": "" }, { - "toolchain": "GCC/4.6.3", "description": "%(gzip_descr)s", "homepage": "http://www.gzip.org/", + "name": "gzip", + "toolchain": "GCC/4.6.3", "version": "1.4", - "versionsuffix": "", - "name": "gzip" + "versionsuffix": "" }, { - "toolchain": "system", "description": "%(gzip_descr)s", "homepage": "http://www.gzip.org/", + "name": "gzip", + "toolchain": "system", "version": "1.4", - "versionsuffix": "", - "name": "gzip" + "versionsuffix": "" }, { - "toolchain": "foss/2018a", "description": "%(gzip_descr)s", "homepage": "http://www.gzip.org/", + "name": "gzip", + "toolchain": "foss/2018a", "version": "1.5", - "versionsuffix": "", - "name": "gzip" + "versionsuffix": "" }, { - "toolchain": "intel/2018a", "description": "%(gzip_descr)s", "homepage": "http://www.gzip.org/", + "name": "gzip", + "toolchain": "intel/2018a", "version": "1.5", - "versionsuffix": "", - "name": "gzip" + "versionsuffix": "" } ]""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} From 62810326b1d8947c108594af831364c9648dd55f Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 6 Dec 2023 21:28:59 +0100 Subject: [PATCH 224/312] more deps --- .../modules/OpenMPI/4.1.5-GCC-12.3.0 | 70 +++++++++++++++++++ .../modules/PMIx/4.2.4-GCCcore-12.3.0 | 0 .../modules/UCC/1.2.0-GCCcore-12.3.0 | 0 .../modules/UCX/1.14.1-GCCcore-12.3.0 | 0 test/framework/modules/gompi/2023a | 38 ++++++++++ .../modules/hwloc/2.9.1-GCCcore-12.3.0 | 0 .../modules/libevent/2.1.12-GCCcore-12.3.0 | 0 7 files changed, 108 insertions(+) create mode 100644 test/framework/modules/OpenMPI/4.1.5-GCC-12.3.0 create mode 100644 test/framework/modules/PMIx/4.2.4-GCCcore-12.3.0 create mode 100644 test/framework/modules/UCC/1.2.0-GCCcore-12.3.0 create mode 100644 test/framework/modules/UCX/1.14.1-GCCcore-12.3.0 create mode 100644 test/framework/modules/gompi/2023a create mode 100644 test/framework/modules/hwloc/2.9.1-GCCcore-12.3.0 create mode 100644 test/framework/modules/libevent/2.1.12-GCCcore-12.3.0 diff --git a/test/framework/modules/OpenMPI/4.1.5-GCC-12.3.0 b/test/framework/modules/OpenMPI/4.1.5-GCC-12.3.0 new file mode 100644 index 0000000000..eb1563c578 --- /dev/null +++ b/test/framework/modules/OpenMPI/4.1.5-GCC-12.3.0 @@ -0,0 +1,70 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +The Open MPI Project is an open source MPI-3 implementation. + + +More information +================ + - Homepage: https://www.open-mpi.org/ + } +} + +module-whatis {Description: The Open MPI Project is an open source MPI-3 implementation.} +module-whatis {Homepage: https://www.open-mpi.org/} +module-whatis {URL: https://www.open-mpi.org/} + +set root /scratch/brussel/vo/000/bvo00005/vsc10009/ebtest/tclmodules/software/OpenMPI/4.1.5-GCC-12.3.0 + +conflict OpenMPI + +if { ![ is-loaded GCC/12.3.0 ] } { + module load GCC/12.3.0 +} + +if { ![ is-loaded zlib/1.2.13-GCCcore-12.3.0 ] } { + module load zlib/1.2.13-GCCcore-12.3.0 +} + +if { ![ is-loaded hwloc/2.9.1-GCCcore-12.3.0 ] } { + module load hwloc/2.9.1-GCCcore-12.3.0 +} + +if { ![ is-loaded libevent/2.1.12-GCCcore-12.3.0 ] } { + module load libevent/2.1.12-GCCcore-12.3.0 +} + +if { ![ is-loaded UCX/1.14.1-GCCcore-12.3.0 ] } { + module load UCX/1.14.1-GCCcore-12.3.0 +} + +if { ![ is-loaded libfabric/1.18.0-GCCcore-12.3.0 ] } { + module load libfabric/1.18.0-GCCcore-12.3.0 +} + +if { ![ is-loaded PMIx/4.2.4-GCCcore-12.3.0 ] } { + module load PMIx/4.2.4-GCCcore-12.3.0 +} + +if { ![ is-loaded UCC/1.2.0-GCCcore-12.3.0 ] } { + module load UCC/1.2.0-GCCcore-12.3.0 +} + +prepend_path("CMAKE_PREFIX_PATH", root) +prepend_path("CPATH", pathJoin(root, "include")) +prepend_path("LD_LIBRARY_PATH", pathJoin(root, "lib")) +prepend_path("LIBRARY_PATH", pathJoin(root, "lib")) +prepend_path("MANPATH", pathJoin(root, "share/man")) +prepend_path("PATH", pathJoin(root, "bin")) +prepend_path("PKG_CONFIG_PATH", pathJoin(root, "lib/pkgconfig")) +prepend_path("XDG_DATA_DIRS", pathJoin(root, "share")) + +setenv EBROOTOPENMPI "$root" +setenv EBVERSIONOPENMPI "4.1.5" +setenv EBDEVELOPENMPI "$root/easybuild/OpenMPI-4.1.5-GCC-12.3.0-easybuild-devel" + +setenv SLURM_MPI_TYPE "pmix" +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/PMIx/4.2.4-GCCcore-12.3.0 b/test/framework/modules/PMIx/4.2.4-GCCcore-12.3.0 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/framework/modules/UCC/1.2.0-GCCcore-12.3.0 b/test/framework/modules/UCC/1.2.0-GCCcore-12.3.0 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/framework/modules/UCX/1.14.1-GCCcore-12.3.0 b/test/framework/modules/UCX/1.14.1-GCCcore-12.3.0 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/framework/modules/gompi/2023a b/test/framework/modules/gompi/2023a new file mode 100644 index 0000000000..c81eeff8df --- /dev/null +++ b/test/framework/modules/gompi/2023a @@ -0,0 +1,38 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +GNU Compiler Collection (GCC) based compiler toolchain, + including OpenMPI for MPI support. + + +More information +================ + - Homepage: (none) + } +} + +module-whatis {Description: GNU Compiler Collection (GCC) based compiler toolchain, + including OpenMPI for MPI support.} +module-whatis {Homepage: (none)} +module-whatis {URL: (none)} + +set root /prefix/software/gompi/2023a + +conflict gompi + +if { ![ is-loaded GCC/12.3.0 ] } { + module load GCC/12.3.0 +} + +if { ![ is-loaded OpenMPI/4.1.5-GCC-12.3.0 ] } { + module load OpenMPI/4.1.5-GCC-12.3.0 +} + +setenv EBROOTGOMPI "$root" +setenv EBVERSIONGOMPI "2023a" +setenv EBDEVELGOMPI "$root/easybuild/gompi-2023a-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/hwloc/2.9.1-GCCcore-12.3.0 b/test/framework/modules/hwloc/2.9.1-GCCcore-12.3.0 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/framework/modules/libevent/2.1.12-GCCcore-12.3.0 b/test/framework/modules/libevent/2.1.12-GCCcore-12.3.0 new file mode 100644 index 0000000000..e69de29bb2 From 0b702ee4cbee3128b3766897dec019ae0bcc9186 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 6 Dec 2023 22:03:52 +0100 Subject: [PATCH 225/312] add magic cookies --- test/framework/modules/PMIx/4.2.4-GCCcore-12.3.0 | 1 + test/framework/modules/UCC/1.2.0-GCCcore-12.3.0 | 1 + test/framework/modules/UCX/1.14.1-GCCcore-12.3.0 | 1 + test/framework/modules/hwloc/2.9.1-GCCcore-12.3.0 | 1 + test/framework/modules/libevent/2.1.12-GCCcore-12.3.0 | 1 + 5 files changed, 5 insertions(+) diff --git a/test/framework/modules/PMIx/4.2.4-GCCcore-12.3.0 b/test/framework/modules/PMIx/4.2.4-GCCcore-12.3.0 index e69de29bb2..1c148cdd28 100644 --- a/test/framework/modules/PMIx/4.2.4-GCCcore-12.3.0 +++ b/test/framework/modules/PMIx/4.2.4-GCCcore-12.3.0 @@ -0,0 +1 @@ +#%Module diff --git a/test/framework/modules/UCC/1.2.0-GCCcore-12.3.0 b/test/framework/modules/UCC/1.2.0-GCCcore-12.3.0 index e69de29bb2..1c148cdd28 100644 --- a/test/framework/modules/UCC/1.2.0-GCCcore-12.3.0 +++ b/test/framework/modules/UCC/1.2.0-GCCcore-12.3.0 @@ -0,0 +1 @@ +#%Module diff --git a/test/framework/modules/UCX/1.14.1-GCCcore-12.3.0 b/test/framework/modules/UCX/1.14.1-GCCcore-12.3.0 index e69de29bb2..1c148cdd28 100644 --- a/test/framework/modules/UCX/1.14.1-GCCcore-12.3.0 +++ b/test/framework/modules/UCX/1.14.1-GCCcore-12.3.0 @@ -0,0 +1 @@ +#%Module diff --git a/test/framework/modules/hwloc/2.9.1-GCCcore-12.3.0 b/test/framework/modules/hwloc/2.9.1-GCCcore-12.3.0 index e69de29bb2..1c148cdd28 100644 --- a/test/framework/modules/hwloc/2.9.1-GCCcore-12.3.0 +++ b/test/framework/modules/hwloc/2.9.1-GCCcore-12.3.0 @@ -0,0 +1 @@ +#%Module diff --git a/test/framework/modules/libevent/2.1.12-GCCcore-12.3.0 b/test/framework/modules/libevent/2.1.12-GCCcore-12.3.0 index e69de29bb2..1c148cdd28 100644 --- a/test/framework/modules/libevent/2.1.12-GCCcore-12.3.0 +++ b/test/framework/modules/libevent/2.1.12-GCCcore-12.3.0 @@ -0,0 +1 @@ +#%Module From ce64b015430e339a86d9bfe8624e2fc774d0615d Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 6 Dec 2023 22:10:54 +0100 Subject: [PATCH 226/312] fix test_avail --- test/framework/modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 7bf87130bd..2d0c52606e 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -197,12 +197,12 @@ def test_avail(self): # test modules include 3 GCC modules and one GCCcore module ms = self.modtool.available('GCC') - expected = ['GCC/4.6.3', 'GCC/4.6.4', 'GCC/6.4.0-2.28', 'GCC/7.3.0-2.30'] + expected = ['GCC/4.6.3', 'GCC/4.6.4', 'GCC/6.4.0-2.28', 'GCC/7.3.0-2.30', 'GCC/12.3.0'] # Tcl-only modules tool does an exact match on module name, Lmod & Tcl/C do prefix matching # EnvironmentModules is a subclass of EnvironmentModulesTcl, but Modules 4+ behaves similarly to Tcl/C impl., # so also append GCCcore/6.2.0 if we are an instance of EnvironmentModules if not isinstance(self.modtool, EnvironmentModulesTcl) or isinstance(self.modtool, EnvironmentModules): - expected.append('GCCcore/6.2.0') + expected.extend(['GCCcore/6.2.0', 'GCCcore/12.3.0']) self.assertEqual(ms, expected) # test modules include one GCC/4.6.3 module From 21c15f579a1f3e14ef1afd9463d59613aba2db91 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 6 Dec 2023 22:44:10 +0100 Subject: [PATCH 227/312] more fixes --- test/framework/modules.py | 4 ++-- test/framework/modules/OpenMPI/4.1.5-GCC-12.3.0 | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 2d0c52606e..916e407d92 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -197,12 +197,12 @@ def test_avail(self): # test modules include 3 GCC modules and one GCCcore module ms = self.modtool.available('GCC') - expected = ['GCC/4.6.3', 'GCC/4.6.4', 'GCC/6.4.0-2.28', 'GCC/7.3.0-2.30', 'GCC/12.3.0'] + expected = ['GCC/12.3.0', 'GCC/4.6.3', 'GCC/4.6.4', 'GCC/6.4.0-2.28', 'GCC/7.3.0-2.30'] # Tcl-only modules tool does an exact match on module name, Lmod & Tcl/C do prefix matching # EnvironmentModules is a subclass of EnvironmentModulesTcl, but Modules 4+ behaves similarly to Tcl/C impl., # so also append GCCcore/6.2.0 if we are an instance of EnvironmentModules if not isinstance(self.modtool, EnvironmentModulesTcl) or isinstance(self.modtool, EnvironmentModules): - expected.extend(['GCCcore/6.2.0', 'GCCcore/12.3.0']) + expected.extend(['GCCcore/12.3.0', 'GCCcore/6.2.0']) self.assertEqual(ms, expected) # test modules include one GCC/4.6.3 module diff --git a/test/framework/modules/OpenMPI/4.1.5-GCC-12.3.0 b/test/framework/modules/OpenMPI/4.1.5-GCC-12.3.0 index eb1563c578..481f81e627 100644 --- a/test/framework/modules/OpenMPI/4.1.5-GCC-12.3.0 +++ b/test/framework/modules/OpenMPI/4.1.5-GCC-12.3.0 @@ -53,14 +53,14 @@ if { ![ is-loaded UCC/1.2.0-GCCcore-12.3.0 ] } { module load UCC/1.2.0-GCCcore-12.3.0 } -prepend_path("CMAKE_PREFIX_PATH", root) -prepend_path("CPATH", pathJoin(root, "include")) -prepend_path("LD_LIBRARY_PATH", pathJoin(root, "lib")) -prepend_path("LIBRARY_PATH", pathJoin(root, "lib")) -prepend_path("MANPATH", pathJoin(root, "share/man")) -prepend_path("PATH", pathJoin(root, "bin")) -prepend_path("PKG_CONFIG_PATH", pathJoin(root, "lib/pkgconfig")) -prepend_path("XDG_DATA_DIRS", pathJoin(root, "share")) +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig +prepend-path XDG_DATA_DIRS $root/share setenv EBROOTOPENMPI "$root" setenv EBVERSIONOPENMPI "4.1.5" From 3f05f7ed5b4545c1acc78817d63d1b692252abeb Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 6 Dec 2023 22:59:21 +0100 Subject: [PATCH 228/312] add another dep --- test/framework/modules/libfabric/1.18.0-GCCcore-12.3.0 | 1 + 1 file changed, 1 insertion(+) create mode 100644 test/framework/modules/libfabric/1.18.0-GCCcore-12.3.0 diff --git a/test/framework/modules/libfabric/1.18.0-GCCcore-12.3.0 b/test/framework/modules/libfabric/1.18.0-GCCcore-12.3.0 new file mode 100644 index 0000000000..1c148cdd28 --- /dev/null +++ b/test/framework/modules/libfabric/1.18.0-GCCcore-12.3.0 @@ -0,0 +1 @@ +#%Module From 646f7a60310757ebb9a15f3d6659fd2c076bf8aa Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 6 Dec 2023 23:47:05 +0100 Subject: [PATCH 229/312] more test fixes --- test/framework/modules.py | 2 +- test/framework/toolchain.py | 131 +++++++++++++++++++++++++----------- 2 files changed, 92 insertions(+), 41 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 916e407d92..4a8a6b15df 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -54,7 +54,7 @@ # number of modules included for testing purposes -TEST_MODULES_COUNT = 92 +TEST_MODULES_COUNT = 110 class ModulesTest(EnhancedTestCase): diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index bee0eec8a9..aacde3baf7 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1840,46 +1840,97 @@ def test_old_new_iccifort(self): scalapack_mt_static_libs_fosscuda = "libscalapack.a,libopenblas.a,libgfortran.a,libpthread.a" scalapack_mt_shared_libs_fosscuda = scalapack_mt_static_libs_fosscuda.replace('.a', '.' + shlib_ext) - for toolc in [('fosscuda', '2018a'), ('foss', '2023a')]: - tc = self.get_toolchain(toolc[0], version=toolc[1]) - tc.prepare() - self.assertEqual(os.environ['BLAS_SHARED_LIBS'], blas_shared_libs_fosscuda) - self.assertEqual(os.environ['BLAS_STATIC_LIBS'], blas_static_libs_fosscuda) - self.assertEqual(os.environ['BLAS_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['BLAS_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) - self.assertEqual(os.environ['LIBBLAS'], libblas_fosscuda) - self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_fosscuda) - - self.assertEqual(os.environ['LAPACK_SHARED_LIBS'], lapack_shared_libs_fosscuda) - self.assertEqual(os.environ['LAPACK_STATIC_LIBS'], lapack_static_libs_fosscuda) - self.assertEqual(os.environ['LAPACK_MT_SHARED_LIBS'], lapack_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['LAPACK_MT_STATIC_LIBS'], lapack_mt_static_libs_fosscuda) - self.assertEqual(os.environ['LIBLAPACK'], liblapack_fosscuda) - self.assertEqual(os.environ['LIBLAPACK_MT'], liblapack_mt_fosscuda) - - self.assertEqual(os.environ['BLAS_LAPACK_SHARED_LIBS'], blas_shared_libs_fosscuda) - self.assertEqual(os.environ['BLAS_LAPACK_STATIC_LIBS'], blas_static_libs_fosscuda) - self.assertEqual(os.environ['BLAS_LAPACK_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['BLAS_LAPACK_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) - - self.assertEqual(os.environ['FFT_SHARED_LIBS'], fft_shared_libs_fosscuda) - self.assertEqual(os.environ['FFT_STATIC_LIBS'], fft_static_libs_fosscuda) - self.assertEqual(os.environ['FFT_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['FFT_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) - self.assertEqual(os.environ['FFTW_SHARED_LIBS'], fft_shared_libs_fosscuda) - self.assertEqual(os.environ['FFTW_STATIC_LIBS'], fft_static_libs_fosscuda) - self.assertEqual(os.environ['FFTW_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['FFTW_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) - self.assertEqual(os.environ['LIBFFT'], libfft_fosscuda) - self.assertEqual(os.environ['LIBFFT_MT'], libfft_mt_fosscuda) - - self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_fosscuda) - self.assertEqual(os.environ['LIBSCALAPACK_MT'], libscalack_mt_fosscuda) - self.assertEqual(os.environ['SCALAPACK_SHARED_LIBS'], scalapack_shared_libs_fosscuda) - self.assertEqual(os.environ['SCALAPACK_STATIC_LIBS'], scalapack_static_libs_fosscuda) - self.assertEqual(os.environ['SCALAPACK_MT_SHARED_LIBS'], scalapack_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['SCALAPACK_MT_STATIC_LIBS'], scalapack_mt_static_libs_fosscuda) - self.modtool.purge() + tc = self.get_toolchain('fosscuda', version='2018a') + tc.prepare() + self.assertEqual(os.environ['BLAS_SHARED_LIBS'], blas_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_STATIC_LIBS'], blas_static_libs_fosscuda) + self.assertEqual(os.environ['BLAS_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBBLAS'], libblas_fosscuda) + self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_fosscuda) + + self.assertEqual(os.environ['LAPACK_SHARED_LIBS'], lapack_shared_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_STATIC_LIBS'], lapack_static_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_MT_SHARED_LIBS'], lapack_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_MT_STATIC_LIBS'], lapack_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBLAPACK'], liblapack_fosscuda) + self.assertEqual(os.environ['LIBLAPACK_MT'], liblapack_mt_fosscuda) + + self.assertEqual(os.environ['BLAS_LAPACK_SHARED_LIBS'], blas_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_STATIC_LIBS'], blas_static_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) + + self.assertEqual(os.environ['FFT_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFT_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBFFT'], libfft_fosscuda) + self.assertEqual(os.environ['LIBFFT_MT'], libfft_mt_fosscuda) + + self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_fosscuda) + self.assertEqual(os.environ['LIBSCALAPACK_MT'], libscalack_mt_fosscuda) + self.assertEqual(os.environ['SCALAPACK_SHARED_LIBS'], scalapack_shared_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_STATIC_LIBS'], scalapack_static_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_MT_SHARED_LIBS'], scalapack_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_MT_STATIC_LIBS'], scalapack_mt_static_libs_fosscuda) + self.modtool.purge() + + tc = self.get_toolchain('foss', version='2023a') + tc.prepare() + self.assertEqual(os.environ['BLAS_SHARED_LIBS'], blas_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['BLAS_STATIC_LIBS'], blas_static_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['BLAS_MT_SHARED_LIBS'], + blas_mt_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['BLAS_MT_STATIC_LIBS'], + blas_mt_static_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LIBBLAS'], libblas_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_fosscuda.replace('openblas', 'flexiblas')) + + self.assertEqual(os.environ['LAPACK_SHARED_LIBS'], lapack_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LAPACK_STATIC_LIBS'], lapack_static_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LAPACK_MT_SHARED_LIBS'], + lapack_mt_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LAPACK_MT_STATIC_LIBS'], + lapack_mt_static_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LIBLAPACK'], liblapack_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LIBLAPACK_MT'], liblapack_mt_fosscuda.replace('openblas', 'flexiblas')) + + self.assertEqual(os.environ['BLAS_LAPACK_SHARED_LIBS'], + blas_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['BLAS_LAPACK_STATIC_LIBS'], + blas_static_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['BLAS_LAPACK_MT_SHARED_LIBS'], + blas_mt_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['BLAS_LAPACK_MT_STATIC_LIBS'], + blas_mt_static_libs_fosscuda.replace('openblas', 'flexiblas')) + + self.assertEqual(os.environ['FFT_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFT_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBFFT'], libfft_fosscuda) + self.assertEqual(os.environ['LIBFFT_MT'], libfft_mt_fosscuda) + + self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LIBSCALAPACK_MT'], libscalack_mt_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['SCALAPACK_SHARED_LIBS'], + scalapack_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['SCALAPACK_STATIC_LIBS'], + scalapack_static_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['SCALAPACK_MT_SHARED_LIBS'], + scalapack_mt_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['SCALAPACK_MT_STATIC_LIBS'], + scalapack_mt_static_libs_fosscuda.replace('openblas', 'flexiblas')) + self.modtool.purge() tc = self.get_toolchain('intel', version='2018a') tc.prepare() From 06461ea123980c124d4fd73fbe6037f6df0b8fe3 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 14 Dec 2023 10:53:15 +0100 Subject: [PATCH 230/312] Introduce `--module-cache-suffix` to allow multiple caches This is useful when having multiple module trees (i.e. different `--installprefix`) to not overwrite the cache file of one with the other when using `--update-modules-tool-cache`. Inspired by how LMod itself creates the cache, e.g. `~/.cache/lmod/spiderT.rapids_x86_64_Linux.lua` --- easybuild/tools/config.py | 1 + easybuild/tools/modules.py | 3 ++- easybuild/tools/options.py | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 6c0a173fe4..ee8eb5fc53 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -239,6 +239,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'job_polling_interval', 'job_target_resource', 'locks_dir', + 'module_cache_suffix', 'modules_footer', 'modules_header', 'mpi_cmd_template', diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 386643e706..f2d74c2137 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1436,7 +1436,8 @@ def update(self): # don't actually update local cache when testing, just return the cache contents return stdout else: - cache_fp = os.path.join(self.USER_CACHE_DIR, 'moduleT.lua') + suffix = build_option('module_cache_suffix') or '' + cache_fp = os.path.join(self.USER_CACHE_DIR, 'moduleT%s.lua' % suffix) self.log.debug("Updating Lmod spider cache %s with output from '%s'" % (cache_fp, ' '.join(cmd))) cache_dir = os.path.dirname(cache_fp) if not os.path.exists(cache_dir): diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index eeba800c09..b840bc4d53 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -461,6 +461,9 @@ def override_options(self): "environment variable and its value separated by a colon (':')", None, 'store', DEFAULT_MINIMAL_BUILD_ENV), 'minimal-toolchains': ("Use minimal toolchain when resolving dependencies", None, 'store_true', False), + 'module-cache-suffix': ("Suffix to add to the cache file name (before the extension) " + "when updating the modules tool cache", + None, 'store', None), 'module-only': ("Only generate module file(s); skip all steps except for %s" % ', '.join(MODULE_ONLY_STEPS), None, 'store_true', False), 'modules-tool-version-check': ("Check version of modules tool being used", None, 'store_true', True), From b4488e0d7083d82ac05f01e4f11c5b32aa4516e1 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 14 Dec 2023 10:40:20 +0100 Subject: [PATCH 231/312] Fix writing spider cache for LMod >= 8.7.12 --- easybuild/tools/modules.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 386643e706..a4c234c352 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1345,7 +1345,6 @@ class Lmod(ModulesTool): DEPR_VERSION = '7.0.0' REQ_VERSION_DEPENDS_ON = '7.6.1' VERSION_REGEXP = r"^Modules\s+based\s+on\s+Lua:\s+Version\s+(?P\d\S*)\s" - USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache') SHOW_HIDDEN_OPTION = '--show-hidden' @@ -1361,7 +1360,14 @@ def __init__(self, *args, **kwargs): setvar('LMOD_EXTENDED_DEFAULT', 'no', verbose=False) super(Lmod, self).__init__(*args, **kwargs) - self.supports_depends_on = StrictVersion(self.version) >= StrictVersion(self.REQ_VERSION_DEPENDS_ON) + version = StrictVersion(self.version) + + self.supports_depends_on = version >= self.REQ_VERSION_DEPENDS_ON + # See https://lmod.readthedocs.io/en/latest/125_personal_spider_cache.html + if version >= '8.7.12': + self.USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.cache', 'lmod') + else: + self.USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache') def check_module_function(self, *args, **kwargs): """Check whether selected module tool matches 'module' function definition.""" From 7f73abf27baf6d2c3aba12ed631bc433f846a143 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 14 Dec 2023 12:34:23 +0100 Subject: [PATCH 232/312] Fix `--inject-checksums` when extension has tuple-format patch `make_list_lines` had a `"'%s'" % x` part where `x` is the patch entry. However that could be a tuple (or even dict) to designate extra options such as the subdir or level where the patch should be applied. This then failed with "TypeError: not all arguments converted during string formatting" Fix by checking the type first and add a test for this case. --- easybuild/framework/easyblock.py | 8 ++++++- test/framework/options.py | 36 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 3ac26eb476..944911cf27 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -4656,8 +4656,14 @@ def inject_checksums(ecs, checksum_type): """ def make_list_lines(values, indent_level): """Make lines for list of values.""" + def to_str(s): + if isinstance(s, string_type): + return "'%s'" % s + else: + return str(s) + line_indent = INDENT_4SPACES * indent_level - return [line_indent + "'%s'," % x for x in values] + return [line_indent + to_str(x) + ',' for x in values] def make_checksum_lines(checksums, indent_level): """Make lines for list of checksums.""" diff --git a/test/framework/options.py b/test/framework/options.py index e6be49ab4f..f3173dbb87 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5948,6 +5948,42 @@ def test_inject_checksums(self): ] self.assertEqual(ec['checksums'], expected_checksums) + # Also works for extensions (all 3 patch formats) + write_file(test_ec, textwrap.dedent(""" + exts_list = [ + ("bar", "0.0", { + 'sources': ['bar-0.0-local.tar.gz'], + 'patches': [ + 'bar-0.0_fix-silly-typo-in-printf-statement.patch', # normal patch + ('bar-0.0_fix-very-silly-typo-in-printf-statement.patch', 0), # patch with patch level + ('toy-0.0_fix-silly-typo-in-printf-statement.patch', 'toy_subdir'), + ], + }), + ] + """), append=True) + self._run_mock_eb(args, raise_error=True, strip=True) + ec = EasyConfigParser(test_ec).get_config_dict() + ext = ec['exts_list'][0] + self.assertEqual((ext[0], ext[1]), ("bar", "0.0")) + ext_opts = ext[2] + expected_patches = [ + 'bar-0.0_fix-silly-typo-in-printf-statement.patch', + ('bar-0.0_fix-very-silly-typo-in-printf-statement.patch', 0), + ('toy-0.0_fix-silly-typo-in-printf-statement.patch', 'toy_subdir') + ] + self.assertEqual(ext_opts['patches'], expected_patches) + expected_checksums = [ + {'bar-0.0-local.tar.gz': + 'f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7'}, + {'bar-0.0_fix-silly-typo-in-printf-statement.patch': + '84db53592e882b5af077976257f9c7537ed971cb2059003fd4faa05d02cae0ab'}, + {'bar-0.0_fix-very-silly-typo-in-printf-statement.patch': + 'd0bf102f9c5878445178c5f49b7cd7546e704c33fe2060c7354b7e473cfeb52b'}, + {'toy-0.0_fix-silly-typo-in-printf-statement.patch': + '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487'} + ] + self.assertEqual(ext_opts['checksums'], expected_checksums) + # passing easyconfig filename as argument to --inject-checksums results in error being reported, # because it's not a valid type of checksum args = ['--inject-checksums', test_ec] From 2816bf8caf273f3e05e69f41f0a0b0788acefb23 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 20 Dec 2023 13:32:11 +0100 Subject: [PATCH 233/312] fix LooseVersion on Python2 --- easybuild/tools/loose_version.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/tools/loose_version.py b/easybuild/tools/loose_version.py index e5594fc5fb..1855fee74a 100644 --- a/easybuild/tools/loose_version.py +++ b/easybuild/tools/loose_version.py @@ -81,6 +81,9 @@ def _cmp(self, other): def __eq__(self, other): return self._cmp(other) == 0 + def __ne__(self, other): + return self._cmp(other) != 0 + def __lt__(self, other): return self._cmp(other) < 0 From 03d02e6e9b8edfa3aec9f50086a5e7908de4a373 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 20 Dec 2023 13:50:10 +0100 Subject: [PATCH 234/312] Enhance tests for LooseVersion --- test/framework/utilities_test.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py index 0aa3a9956e..ba4766f302 100644 --- a/test/framework/utilities_test.py +++ b/test/framework/utilities_test.py @@ -123,6 +123,15 @@ def test_LooseVersion(self): self.assertLess(LooseVersion('1.02'), '2.01') self.assertLessEqual('1.02', LooseVersion('2.01')) self.assertLessEqual(LooseVersion('1.02'), '2.01') + # Negation of all ops, i.e. verify each op can return False + self.assertFalse(LooseVersion('2.02') != '2.02') + self.assertFalse(LooseVersion('2.02') <= '2.01') + self.assertFalse(LooseVersion('2.02') < '2.01') + self.assertFalse(LooseVersion('2.02') < '2.02') + self.assertFalse(LooseVersion('2.02') == '2.03') + self.assertFalse(LooseVersion('2.02') >= '2.03') + self.assertFalse(LooseVersion('2.02') > '2.03') + self.assertFalse(LooseVersion('2.02') > '2.02') # Some comparisons we might do: Full version on left hand side, shorter on right self.assertGreater(LooseVersion('2.1.5'), LooseVersion('2.1')) @@ -135,7 +144,7 @@ def test_LooseVersion(self): self.assertGreater(LooseVersion('1.0'), LooseVersion('1')) self.assertLess(LooseVersion('1'), LooseVersion('1.0')) - # The following test is taken from Python disutils tests + # The following test is taken from Python distutils tests # licensed under the Python Software Foundation License Version 2 versions = (('1.5.1', '1.5.2b2', -1), ('161', '3.10a', 1), From b1c5d082e1940b81da947203779937edb10bb993 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 21 Dec 2023 10:22:58 +0100 Subject: [PATCH 235/312] Remove superflous string formatting `"%s" % var` is the same as `str(var)` while the latter is faster due to less parsing, inserting etc. In some places `var` was already a string so it can just be used. --- easybuild/base/generaloption.py | 8 ++++---- easybuild/framework/easyconfig/format/format.py | 2 +- easybuild/framework/easyconfig/tweak.py | 2 +- easybuild/tools/docs.py | 6 +++--- easybuild/tools/filetools.py | 2 +- easybuild/tools/module_generator.py | 2 +- easybuild/tools/testing.py | 2 +- test/framework/module_generator.py | 8 ++++---- test/framework/options.py | 2 +- .../module_naming_scheme/test_module_naming_scheme.py | 4 ++-- test/framework/toy_build.py | 4 ++-- 11 files changed, 21 insertions(+), 21 deletions(-) diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index 51f7daafe3..6a4c5f0d5d 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -90,7 +90,7 @@ def set_columns(cols=None): pass if cols is not None: - os.environ['COLUMNS'] = "%s" % cols + os.environ['COLUMNS'] = str(cols) def what_str_list_tuple(name): @@ -822,8 +822,8 @@ def get_env_options(self): self.environment_arguments.append("%s=%s" % (lo, val)) else: # interpretation of values: 0/no/false means: don't set it - if ("%s" % val).lower() not in ("0", "no", "false",): - self.environment_arguments.append("%s" % lo) + if str(val).lower() not in ("0", "no", "false",): + self.environment_arguments.append(str(lo)) else: self.log.debug("Environment variable %s is not set" % env_opt_name) @@ -1189,7 +1189,7 @@ def add_group_parser(self, opt_dict, description, prefix=None, otherdefaults=Non for extra_detail in details[4:]: if isinstance(extra_detail, (list, tuple,)): # choices - nameds['choices'] = ["%s" % x for x in extra_detail] # force to strings + nameds['choices'] = [str(x) for x in extra_detail] # force to strings hlp += ' (choices: %s)' % ', '.join(nameds['choices']) elif isinstance(extra_detail, string_type) and len(extra_detail) == 1: args.insert(0, "-%s" % extra_detail) diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index 8abb697ece..9a1626c60b 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -370,7 +370,7 @@ def parse(self, configobj): for key, value in self.supported.items(): if key not in self.VERSION_OPERATOR_VALUE_TYPES: raise EasyBuildError('Unsupported key %s in %s section', key, self.SECTION_MARKER_SUPPORTED) - self.sections['%s' % key] = value + self.sections[key] = value for key, supported_key, fn_name in [('version', 'versions', 'get_version_str'), ('toolchain', 'toolchains', 'as_dict')]: diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index c3e26ab868..7aaa69b69a 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -324,7 +324,7 @@ def __repr__(self): newval = "%s + %s" % (fval, res.group('val')) _log.debug("Prepending %s to %s" % (fval, key)) else: - newval = "%s" % fval + newval = str(fval) _log.debug("Overwriting %s with %s" % (key, fval)) ectxt = regexp.sub("%s = %s" % (res.group('key'), newval), ectxt) _log.info("Tweaked %s list to '%s'" % (key, newval)) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 4f4bfca99c..4fcf1ad466 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -289,7 +289,7 @@ def avail_easyconfig_licenses_md(): lics = sorted(EASYCONFIG_LICENSES_DICT.items()) table_values = [ ["``%s``" % lic().name for _, lic in lics], - ["%s" % lic().description for _, lic in lics], + [lic().description for _, lic in lics], ["``%s``" % lic().version for _, lic in lics], ] @@ -1199,7 +1199,7 @@ def avail_toolchain_opts_md(name, tc_dict): tc_items = sorted(tc_dict.items()) table_values = [ ['``%s``' % val[0] for val in tc_items], - ['%s' % val[1][1] for val in tc_items], + [val[1][1] for val in tc_items], ['``%s``' % val[1][0] for val in tc_items], ] @@ -1217,7 +1217,7 @@ def avail_toolchain_opts_rst(name, tc_dict): tc_items = sorted(tc_dict.items()) table_values = [ ['``%s``' % val[0] for val in tc_items], - ['%s' % val[1][1] for val in tc_items], + [val[1][1] for val in tc_items], ['``%s``' % val[1][0] for val in tc_items], ] diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 6e4bd2d83e..5a2f724bc8 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2668,7 +2668,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): clone_cmd.append('%s/%s.git' % (url, repo_name)) if clone_into: - clone_cmd.append('%s' % clone_into) + clone_cmd.append(clone_into) tmpdir = tempfile.mkdtemp() cwd = change_dir(tmpdir) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 0c19b93c8b..45582bbe5c 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -1289,7 +1289,7 @@ def get_description(self, conflict=True): extensions_list = self._generate_extensions_list() if extensions_list: - extensions_stmt = 'extensions("%s")' % ','.join(['%s' % x for x in extensions_list]) + extensions_stmt = 'extensions("%s")' % ','.join([str(x) for x in extensions_list]) # put this behind a Lmod version check as 'extensions' is only (well) supported since Lmod 8.2.8, # see https://lmod.readthedocs.io/en/latest/330_extensions.html#module-extensions and # https://github.com/TACC/Lmod/issues/428 diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index 68eedc23ce..ffd8ce580b 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -178,7 +178,7 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nrs=None, gist_ ]) test_report.extend([ "#### Test result", - "%s" % msg, + msg, "", ]) diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index a0c6131140..d2fbb6f642 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -82,7 +82,7 @@ def test_descr(self): '', 'Description', '===========', - "%s" % descr, + descr, '', '', "More information", @@ -107,7 +107,7 @@ def test_descr(self): '', 'Description', '===========', - "%s" % descr, + descr, '', '', "More information", @@ -137,7 +137,7 @@ def test_descr(self): '', 'Description', '===========', - "%s" % descr, + descr, '', '', "More information", @@ -161,7 +161,7 @@ def test_descr(self): '', 'Description', '===========', - "%s" % descr, + descr, '', '', "More information", diff --git a/test/framework/options.py b/test/framework/options.py index f3173dbb87..3bfc863e41 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2190,7 +2190,7 @@ def test_ignore_osdeps(self): self.assertTrue(regex.search(outtxt), "OS dependencies are checked, outtxt: %s" % outtxt) msg = "One or more OS dependencies were not found: " msg += r"\[\('nosuchosdependency',\), \('nosuchdep_option1', 'nosuchdep_option2'\)\]" - regex = re.compile(r'%s' % msg, re.M) + regex = re.compile(msg, re.M) self.assertTrue(regex.search(outtxt), "OS dependencies are honored, outtxt: %s" % outtxt) # check whether OS dependencies are effectively ignored diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py index 5eb5f66682..02934e5745 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py @@ -64,6 +64,6 @@ def det_module_symlink_paths(self, ec): def is_short_modname_for(self, modname, name): """ - Determine whether the specified (short) module name is a module for software with the specified name. + Determine whether the specified (short) module name is a moduleq for software with the specified name. """ - return modname.find('%s' % name) != -1 + return modname.find(name) != -1 diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 9ccb3c08df..cc84ce86d1 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1491,7 +1491,7 @@ def test_toy_module_fulltxt(self): mod_txt_regex_pattern = '\n'.join([ r'help\(\[==\[', r'', - r'%s' % help_txt, + help_txt, r'\]==\]\)', r'', r'whatis\(\[==\[Description: Toy C program, 100% toy.\]==\]\)', @@ -1528,7 +1528,7 @@ def test_toy_module_fulltxt(self): r'proc ModulesHelp { } {', r' puts stderr {', r'', - r'%s' % help_txt, + help_txt, r' }', r'}', r'', From a748eaf2e8f65f73b0e8557162567143de09f11a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 21 Dec 2023 11:11:07 +0100 Subject: [PATCH 236/312] Show empty description for license instead of "None" --- easybuild/tools/docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 4fcf1ad466..1d049a19c1 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -289,7 +289,7 @@ def avail_easyconfig_licenses_md(): lics = sorted(EASYCONFIG_LICENSES_DICT.items()) table_values = [ ["``%s``" % lic().name for _, lic in lics], - [lic().description for _, lic in lics], + [lic().description or '' for _, lic in lics], ["``%s``" % lic().version for _, lic in lics], ] From 13191968a1b43436c5df63d8cd52edc8d503560c Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 21 Dec 2023 13:41:47 +0100 Subject: [PATCH 237/312] Remove typo Co-authored-by: Kenneth Hoste --- .../tools/module_naming_scheme/test_module_naming_scheme.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py index 02934e5745..5d8dec26c8 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py @@ -64,6 +64,6 @@ def det_module_symlink_paths(self, ec): def is_short_modname_for(self, modname, name): """ - Determine whether the specified (short) module name is a moduleq for software with the specified name. + Determine whether the specified (short) module name is a module for software with the specified name. """ return modname.find(name) != -1 From 3830181bb4a6fad4f8d3f71f884961345fc47eae Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 21 Dec 2023 16:13:31 +0100 Subject: [PATCH 238/312] Cleanup uses of `getattr` and `hasattr` `hasattr` is basically calling `getattr` and checking for `AttributeError`. It also supports a default value in case the attribute is not found. Make use of that default value and catch the exception directly instead of checking `hasattr` where that makes sense. --- easybuild/base/fancylogger.py | 2 +- easybuild/base/generaloption.py | 2 +- easybuild/base/optcomplete.py | 21 ++++++++-------- .../easyconfig/format/pyheaderconfigobj.py | 11 +++++---- easybuild/tools/configobj.py | 2 +- easybuild/tools/docs.py | 7 +++--- easybuild/tools/github.py | 4 ++-- easybuild/tools/systemtools.py | 4 ++-- easybuild/tools/toolchain/linalg.py | 10 ++++---- easybuild/tools/toolchain/toolchain.py | 24 +++++++++++-------- easybuild/tools/toolchain/utilities.py | 11 +++++---- test/framework/systemtools.py | 7 ++---- 12 files changed, 53 insertions(+), 52 deletions(-) diff --git a/easybuild/base/fancylogger.py b/easybuild/base/fancylogger.py index b5a63477c6..93582b0cf1 100644 --- a/easybuild/base/fancylogger.py +++ b/easybuild/base/fancylogger.py @@ -286,7 +286,7 @@ def makeRecord(self, name, level, pathname, lineno, msg, args, excinfo, func=Non overwrite make record to use a fancy record (with more options) """ logrecordcls = logging.LogRecord - if hasattr(self, 'fancyrecord') and self.fancyrecord: + if getattr(self, 'fancyrecord', None): logrecordcls = FancyLogRecord try: new_msg = str(msg) diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index 51f7daafe3..49b1168008 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -1031,7 +1031,7 @@ def main_options(self): # make_init is deprecated if hasattr(self, 'make_init'): self.log.debug('main_options: make_init is deprecated. Rename function to main_options.') - getattr(self, 'make_init')() + self.make_init() else: # function names which end with _options and do not start with main or _ reg_main_options = re.compile("^(?!_|main).*_options$") diff --git a/easybuild/base/optcomplete.py b/easybuild/base/optcomplete.py index 9a2bc8a127..3fa75a6635 100644 --- a/easybuild/base/optcomplete.py +++ b/easybuild/base/optcomplete.py @@ -513,14 +513,15 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete if option: if option.nargs > 0: optarg = True - if hasattr(option, 'completer'): + try: completer = option.completer - elif option.choices: - completer = ListCompleter(option.choices) - elif option.type in ('string',): - completer = opt_completer - else: - completer = NoneCompleter() + except AttributeError: + if option.choices: + completer = ListCompleter(option.choices) + elif option.type in ('string',): + completer = opt_completer + else: + completer = NoneCompleter() # Warn user at least, it could help him figure out the problem. elif hasattr(option, 'completer'): msg = "Error: optparse option with a completer does not take arguments: %s" % (option) @@ -616,11 +617,9 @@ class CmdComplete(object): def autocomplete(self, completer=None): parser = OPTIONPARSER_CLASS(self.__doc__.strip()) if hasattr(self, 'addopts'): - fnc = getattr(self, 'addopts') - fnc(parser) + self.addopts(parser) - if hasattr(self, 'completer'): - completer = getattr(self, 'completer') + completer = getattr(self, 'completer', completer) return autocomplete(parser, completer) diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py index b9fd31bf92..37e14b529e 100644 --- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py +++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py @@ -233,12 +233,13 @@ def pyheader_env(self): current_builtins = globals()['__builtins__'] builtins = {} for name in self.PYHEADER_ALLOWED_BUILTINS: - if hasattr(current_builtins, name): + try: builtins[name] = getattr(current_builtins, name) - elif isinstance(current_builtins, dict) and name in current_builtins: - builtins[name] = current_builtins[name] - else: - self.log.warning('No builtin %s found.' % name) + except AttributeError: + if isinstance(current_builtins, dict) and name in current_builtins: + builtins[name] = current_builtins[name] + else: + self.log.warning('No builtin %s found.' % name) global_vars['__builtins__'] = builtins self.log.debug("Available builtins: %s" % global_vars['__builtins__']) diff --git a/easybuild/tools/configobj.py b/easybuild/tools/configobj.py index 48c7dd348d..6ac667b5bd 100644 --- a/easybuild/tools/configobj.py +++ b/easybuild/tools/configobj.py @@ -1254,7 +1254,7 @@ def set_section(in_section, this_section): self.configspec = None return - elif getattr(infile, 'read', MISSING) is not MISSING: + elif hasattr(infile, 'read'): # This supports file like objects infile = infile.read() or [] # needs splitting into lines - but needs doing *after* decoding diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 4f4bfca99c..dc707466ad 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1182,10 +1182,9 @@ def avail_toolchain_opts(name, output_format=FORMAT_TXT): tc_dict = {} for cst in ['COMPILER_SHARED_OPTS', 'COMPILER_UNIQUE_OPTS', 'MPI_SHARED_OPTS', 'MPI_UNIQUE_OPTS']: - if hasattr(tc, cst): - opts = getattr(tc, cst) - if opts is not None: - tc_dict.update(opts) + opts = getattr(tc, cst, None) + if opts is not None: + tc_dict.update(opts) return generate_doc('avail_toolchain_opts_%s' % output_format, [name, tc_dict]) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 983dc977c4..ca013743d7 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -719,9 +719,9 @@ def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=Fa raise EasyBuildError("Fetching branch '%s' from remote %s failed: empty result", branch_name, origin) # git checkout -b ; git pull - if hasattr(origin.refs, branch_name): + try: origin_branch = getattr(origin.refs, branch_name) - else: + except AttributeError: raise EasyBuildError("Branch '%s' not found at %s", branch_name, github_url) _log.debug("Checking out branch '%s' from remote %s", branch_name, github_url) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 78ba312203..59b8728454 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -1340,8 +1340,8 @@ def det_pypkg_version(pkg_name, imported_pkg, import_name=None): except pkg_resources.DistributionNotFound as err: _log.debug("%s Python package not found: %s", pkg_name, err) - if version is None and hasattr(imported_pkg, '__version__'): - version = imported_pkg.__version__ + if version is None: + version = getattr(imported_pkg, '__version__', None) return version diff --git a/easybuild/tools/toolchain/linalg.py b/easybuild/tools/toolchain/linalg.py index b7ce694ff7..907993571c 100644 --- a/easybuild/tools/toolchain/linalg.py +++ b/easybuild/tools/toolchain/linalg.py @@ -212,9 +212,9 @@ def _set_blacs_variables(self): """Set BLACS related variables""" lib_map = {} - if hasattr(self, 'BLAS_LIB_MAP') and self.BLAS_LIB_MAP is not None: + if getattr(self, 'BLAS_LIB_MAP', None) is not None: lib_map.update(self.BLAS_LIB_MAP) - if hasattr(self, 'BLACS_LIB_MAP') and self.BLACS_LIB_MAP is not None: + if getattr(self, 'BLACS_LIB_MAP', None) is not None: lib_map.update(self.BLACS_LIB_MAP) # BLACS @@ -254,11 +254,11 @@ def _set_scalapack_variables(self): raise EasyBuildError("_set_blas_variables: SCALAPACK_LIB not set") lib_map = {} - if hasattr(self, 'BLAS_LIB_MAP') and self.BLAS_LIB_MAP is not None: + if getattr(self, 'BLAS_LIB_MAP', None) is not None: lib_map.update(self.BLAS_LIB_MAP) - if hasattr(self, 'BLACS_LIB_MAP') and self.BLACS_LIB_MAP is not None: + if getattr(self, 'BLACS_LIB_MAP', None) is not None: lib_map.update(self.BLACS_LIB_MAP) - if hasattr(self, 'SCALAPACK_LIB_MAP') and self.SCALAPACK_LIB_MAP is not None: + if getattr(self, 'SCALAPACK_LIB_MAP', None) is not None: lib_map.update(self.SCALAPACK_LIB_MAP) self.SCALAPACK_LIB = self.variables.nappend('LIBSCALAPACK_ONLY', [x % lib_map for x in self.SCALAPACK_LIB]) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index a16fe35dc0..4c30faf19a 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -163,13 +163,13 @@ def _is_toolchain_for(cls, name): """see if this class can provide support for toolchain named name""" # TODO report later in the initialization the found version if name: - if hasattr(cls, 'NAME') and name == cls.NAME: - return True - else: + try: + return name == cls.NAME + except AttributeError: return False else: # is no name is supplied, check whether class can be used as a toolchain - return hasattr(cls, 'NAME') and cls.NAME + return bool(getattr(cls, 'NAME', None)) _is_toolchain_for = classmethod(_is_toolchain_for) @@ -330,10 +330,12 @@ def _copy_class_constants(self): if key not in self.CLASS_CONSTANT_COPIES: self.CLASS_CONSTANT_COPIES[key] = {} for cst in self.CLASS_CONSTANTS_TO_RESTORE: - if hasattr(self, cst): - self.CLASS_CONSTANT_COPIES[key][cst] = copy.deepcopy(getattr(self, cst)) - else: + try: + value = getattr(self, cst) + except AttributeError: raise EasyBuildError("Class constant '%s' to be restored does not exist in %s", cst, self) + else: + self.CLASS_CONSTANT_COPIES[key][cst] = copy.deepcopy(value) self.log.devel("Copied class constants: %s", self.CLASS_CONSTANT_COPIES[key]) @@ -342,10 +344,12 @@ def _restore_class_constants(self): key = self.__class__ for cst in self.CLASS_CONSTANT_COPIES[key]: newval = copy.deepcopy(self.CLASS_CONSTANT_COPIES[key][cst]) - if hasattr(self, cst): - self.log.devel("Restoring class constant '%s' to %s (was: %s)", cst, newval, getattr(self, cst)) - else: + try: + oldval = getattr(self, cst) + except AttributeError: self.log.devel("Restoring (currently undefined) class constant '%s' to %s", cst, newval) + else: + self.log.devel("Restoring class constant '%s' to %s (was: %s)", cst, newval, oldval) setattr(self, cst, newval) diff --git a/easybuild/tools/toolchain/utilities.py b/easybuild/tools/toolchain/utilities.py index 48a586c567..56242b8fe4 100644 --- a/easybuild/tools/toolchain/utilities.py +++ b/easybuild/tools/toolchain/utilities.py @@ -63,7 +63,7 @@ def search_toolchain(name): package = easybuild.tools.toolchain check_attr_name = '%s_PROCESSED' % TC_CONST_PREFIX - if not hasattr(package, check_attr_name) or not getattr(package, check_attr_name): + if not getattr(package, check_attr_name, None): # import all available toolchains, so we know about them tc_modules = import_available_modules('easybuild.toolchains') @@ -76,7 +76,7 @@ def search_toolchain(name): if hasattr(elem, '__module__'): # exclude the toolchain class defined in that module if not tc_mod.__file__ == sys.modules[elem.__module__].__file__: - elem_name = elem.__name__ if hasattr(elem, '__name__') else elem + elem_name = getattr(elem, '__name__', elem) _log.debug("Adding %s to list of imported classes used for looking for constants", elem_name) mod_classes.append(elem) @@ -89,13 +89,14 @@ def search_toolchain(name): tc_const_value = getattr(mod_class_mod, elem) _log.debug("Found constant %s ('%s') in module %s, adding it to %s", tc_const_name, tc_const_value, mod_class_mod.__name__, package.__name__) - if hasattr(package, tc_const_name): + try: cur_value = getattr(package, tc_const_name) + except AttributeError: + setattr(package, tc_const_name, tc_const_value) + else: if not tc_const_value == cur_value: raise EasyBuildError("Constant %s.%s defined as '%s', can't set it to '%s'.", package.__name__, tc_const_name, cur_value, tc_const_value) - else: - setattr(package, tc_const_name, tc_const_value) # indicate that processing of toolchain constants is done, so it's not done again setattr(package, check_attr_name, True) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 1707d6ec54..45f51991ff 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -366,17 +366,14 @@ def setUp(self): self.orig_is_readable = st.is_readable self.orig_read_file = st.read_file self.orig_run_cmd = st.run_cmd - self.orig_platform_dist = st.platform.dist if hasattr(st.platform, 'dist') else None + self.orig_platform_dist = getattr(st.platform, 'dist', None) self.orig_platform_uname = st.platform.uname self.orig_get_tool_version = st.get_tool_version self.orig_sys_version_info = st.sys.version_info self.orig_HAVE_ARCHSPEC = st.HAVE_ARCHSPEC self.orig_HAVE_DISTRO = st.HAVE_DISTRO self.orig_ETC_OS_RELEASE = st.ETC_OS_RELEASE - if hasattr(st, 'archspec_cpu_host'): - self.orig_archspec_cpu_host = st.archspec_cpu_host - else: - self.orig_archspec_cpu_host = None + self.orig_archspec_cpu_host = getattr(st, 'archspec_cpu_host', None) def tearDown(self): """Cleanup after systemtools test.""" From 5aec5e21be38c515f3b3f944920ff86067f86ecb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Dec 2023 16:44:50 +0100 Subject: [PATCH 239/312] use more recent easyblocks PR in test_github_merge_pr --- test/framework/options.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index f3173dbb87..4b5e1afce8 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4822,7 +4822,7 @@ def test_github_merge_pr(self): # --merge-pr also works on easyblocks (& framework) PRs args = [ '--merge-pr', - '2805', + '2995', '--pr-target-repo=easybuild-easyblocks', '-D', '--github-user=%s' % GITHUB_TEST_ACCOUNT, @@ -4830,12 +4830,12 @@ def test_github_merge_pr(self): stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) self.assertEqual(stderr.strip(), '') expected_stdout = '\n'.join([ - "Checking eligibility of easybuilders/easybuild-easyblocks PR #2805 for merging...", + "Checking eligibility of easybuilders/easybuild-easyblocks PR #2995 for merging...", "* targets develop branch: OK", "* test suite passes: OK", "* no pending change requests: OK", - "* approved review: OK (by ocaisa)", - "* milestone is set: OK (4.6.2)", + "* approved review: OK (by boegel)", + "* milestone is set: OK (4.8.1)", "* mergeable state is clean: PR is already merged", '', "Review OK, merging pull request!", From 1fad79e11d714a66acd7afa747d461e48ce60399 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Mon, 30 Oct 2023 20:52:30 +0100 Subject: [PATCH 240/312] Adapt VERSION_REGEXP for EnvironmentModules When using an Environment Modules version built from git repository, version number contains git branch name, number of commit since last released version and commit hash. This commit adapts VERSION_REGEXP for EnvironmentModules class to allow using development or locally adapted versions of Environment Modules with EasyBuild. Fixes #4126 --- easybuild/tools/modules.py | 2 +- test/framework/modulestool.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 7d7181fe30..bdac20c274 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1326,7 +1326,7 @@ class EnvironmentModules(EnvironmentModulesTcl): COMMAND_ENVIRONMENT = 'MODULES_CMD' REQ_VERSION = '4.0.0' MAX_VERSION = None - VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d\S*)\s' + VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d[^+\s]*)(\+\S*)?\s' def __init__(self, *args, **kwargs): """Constructor, set Environment Modules-specific class variable values.""" diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index fb991ad797..f43a91e3b3 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -209,6 +209,21 @@ def test_environment_modules_specific(self): mt = EnvironmentModules(testing=True) self.assertIsInstance(mt.loaded_modules(), list) # dummy usage + # initialize Environment Modules tool with non-official version number + # pass (fake) full path to 'modulecmd.tcl' via $MODULES_CMD + fake_path = os.path.join(self.test_installpath, 'libexec', 'modulecmd.tcl') + fake_modulecmd_txt = '\n'.join([ + 'puts stderr {Modules Release 5.3.1+unload-188-g14b6b59b (2023-10-21)}', + "puts {os.environ['FOO'] = 'foo'}", + ]) + write_file(fake_path, fake_modulecmd_txt) + os.chmod(fake_path, stat.S_IRUSR | stat.S_IXUSR) + os.environ['_module_raw'] = "() { eval `%s' bash $*`;\n}" % fake_path + os.environ['MODULES_CMD'] = fake_path + EnvironmentModules.COMMAND = fake_path + mt = EnvironmentModules(testing=True) + self.assertTrue(os.path.samefile(mt.cmd, fake_path), "%s - %s" % (mt.cmd, fake_path)) + def tearDown(self): """Testcase cleanup.""" super(ModulesToolTest, self).tearDown() From a3cb2ccdf6a8ead993d553e00a637ad747d1c5ff Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Sun, 29 Oct 2023 13:23:23 +0100 Subject: [PATCH 241/312] Get available hidden modules on EnvironmentModules Environment Modules option "--all" was introduced in version 4.6.0 to obtain hidden modules among "module avail" results. Update EnvironmentModules class to make use of this option to return hidden modules on "available" function like done on Lmod class. Update "test_avail" unit test to match new results obtain with Environment Modules 4.6+ --- easybuild/tools/modules.py | 21 +++++++++++++++++++-- test/framework/modules.py | 5 +++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 7d7181fe30..d3c6019aba 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1283,14 +1283,14 @@ def tweak_stdout(txt): return super(EnvironmentModulesTcl, self).run_module(*args, **kwargs) - def available(self, mod_name=None): + def available(self, mod_name=None, extra_args=None): """ Return a list of available modules for the given (partial) module name; use None to obtain a list of all available modules. :param mod_name: a (partial) module name for filtering (default: None) """ - mods = super(EnvironmentModulesTcl, self).available(mod_name=mod_name) + mods = super(EnvironmentModulesTcl, self).available(mod_name=mod_name, extra_args=extra_args) # strip off slash at beginning, if it's there # under certain circumstances, 'modulecmd.tcl avail' (DEISA variant) spits out available modules like this clean_mods = [mod.lstrip(os.path.sep) for mod in mods] @@ -1328,6 +1328,8 @@ class EnvironmentModules(EnvironmentModulesTcl): MAX_VERSION = None VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d\S*)\s' + SHOW_HIDDEN_OPTION = '--all' + def __init__(self, *args, **kwargs): """Constructor, set Environment Modules-specific class variable values.""" # ensure in-depth modulepath search (MODULES_AVAIL_INDEPTH has been introduced in v4.3) @@ -1383,6 +1385,21 @@ def check_module_output(self, cmd, stdout, stderr): else: self.log.debug("No errors detected when running module command '%s'", cmd) + def available(self, mod_name=None, extra_args=None): + """ + Return a list of available modules for the given (partial) module name; + use None to obtain a list of all available modules. + + :param mod_name: a (partial) module name for filtering (default: None) + """ + if extra_args is None: + extra_args = [] + # make hidden modules visible (requires Environment Modules 4.6.0) + if StrictVersion(self.version) >= StrictVersion('4.6.0'): + extra_args.append(self.SHOW_HIDDEN_OPTION) + + return super(EnvironmentModules, self).available(mod_name=mod_name, extra_args=extra_args) + def get_setenv_value_from_modulefile(self, mod_name, var_name): """ Get value for specific 'setenv' statement from module file for the specified module. diff --git a/test/framework/modules.py b/test/framework/modules.py index c2460c7a11..9151527aac 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -218,6 +218,11 @@ def test_avail(self): self.assertIn('bzip2/.1.0.6', ms) self.assertIn('toy/.0.0-deps', ms) self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) + elif (isinstance(self.modtool, EnvironmentModules) + and StrictVersion(self.modtool.version) >= StrictVersion('4.6.0')): + self.assertEqual(len(ms), TEST_MODULES_COUNT + 2) + self.assertIn('toy/.0.0-deps', ms) + self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) else: self.assertEqual(len(ms), TEST_MODULES_COUNT) From 86971782b2d4329efb2caeb78af72a0bb9a0cc5e Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Fri, 29 Dec 2023 15:49:19 +0100 Subject: [PATCH 242/312] prepare release notes for EasyBuild v4.9.0 + bump version to 4.9.0 --- RELEASE_NOTES | 27 +++++++++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index a43ce28535..641a0d70d6 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,33 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.9.0 (30 December 2023) +------------------------- + +feature release + +- various enhancements, including: + - allow tweaking of easyconfigs from different toolchains (#3669) + - add `--module-cache-suffix` configuration setting to allow multiple (Lmod) caches (#4403) +- various bug fixes, including: + - deduplicate warnings & errors found in logs and add initial newline + tab in output (#4361) + - fix support for Environment Modules as modules tool to pass unit tests with v4.2+ (#4369) + - Adapt module function check for Environment Modules v4+ (#4371) + - only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9 (#4375) + - Use `-qopenmp` instead of `-fiopenmp` for OpenMP in Intel compilers (#4377) + - fix `LIBBLAS_MT` for FlexiBLAS, ensure `-lpthread` is included (#4379) + - relax major version match regex in `find_related_easyconfigs` using for `--review-pr` (#4385) + - eliminate duplicate multideps from generated module files (#4386) + - resolve templated values in extension names in `_make_extension_list` (#4392) + - use source toolchain version when passing only `--try-toolchain` (#4395) + - fix writing spider cache for Lmod >= 8.7.12 (#4402) + - fix `--inject-checksums` when extension specifies patch file in tuple format (#4405) + - fix LooseVersion on Python 2.7 (#4408) + - use more recent easyblocks PR in `test_github_merge_pr` (#4414) +- other changes: + - extend test that checks build environment to recent `foss/2023a` toolchain (#4391) + + v4.8.2 (29 October 2023) ------------------------ diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 655f7d0a49..e3d5feb017 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -45,7 +45,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.9.0.dev0') +VERSION = LooseVersion('4.9.0') UNKNOWN = 'UNKNOWN' From 4288f5b1274fc4afe362ecbda73e0442520745a7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 29 Dec 2023 17:45:03 +0100 Subject: [PATCH 243/312] minor code style fixes in list_software_json --- easybuild/tools/docs.py | 26 +++++++++++++++----------- easybuild/tools/options.py | 4 ++-- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 43f6038b26..199cd22d78 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -38,8 +38,8 @@ """ import copy import inspect -import os import json +import os from easybuild.tools import LooseVersion from easybuild.base import fancylogger @@ -73,10 +73,10 @@ DETAILED = 'detailed' SIMPLE = 'simple' +FORMAT_JSON = 'json' FORMAT_MD = 'md' FORMAT_RST = 'rst' FORMAT_TXT = 'txt' -FORMAT_JSON = 'json' def generate_doc(name, params): @@ -1065,22 +1065,26 @@ def list_software_json(software, detailed=False): """ lines = ['['] for key in sorted(software, key=lambda x: x.lower()): - for tmp in software[key]: + for entry in software[key]: if detailed: # deep copy here to avoid modifying the original dict - x = copy.deepcopy(tmp) - x['description'] = ' '.join(x['description'].split('\n')).strip() + entry = copy.deepcopy(entry) + entry['description'] = ' '.join(entry['description'].split('\n')).strip() else: - x = {} - x['name'] = key + entry = {} + entry['name'] = key - lines.append(json.dumps(x, indent=4, sort_keys=True, separators=(',', ': ')) + ",") + lines.append(json.dumps(entry, indent=4, sort_keys=True, separators=(',', ': ')) + ",") if not detailed: break - # remove last line last comma + + # remove trailing comma on last line if len(lines) > 1: - lines[-1] = lines[-1][:-1] - return '\n'.join(lines) + '\n]' + lines[-1] = lines[-1].rstrip(',') + + lines.append(']') + + return '\n'.join(lines) def list_toolchains(output_format=FORMAT_TXT): diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 64c46b61b5..fec9bb5da6 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -79,7 +79,7 @@ from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path from easybuild.tools.config import BuildOptions, ConfigurationVariables from easybuild.tools.configobj import ConfigObj, ConfigObjError -from easybuild.tools.docs import FORMAT_MD, FORMAT_RST, FORMAT_TXT, FORMAT_JSON +from easybuild.tools.docs import FORMAT_JSON, FORMAT_MD, FORMAT_RST, FORMAT_TXT from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses from easybuild.tools.docs import avail_toolchain_opts, avail_easyconfig_params, avail_easyconfig_templates from easybuild.tools.docs import list_easyblocks, list_toolchains @@ -470,7 +470,7 @@ def override_options(self): 'optarch': ("Set architecture optimization, overriding native architecture optimizations", None, 'store', None), 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, - [FORMAT_MD, FORMAT_RST, FORMAT_TXT, FORMAT_JSON]), + [FORMAT_JSON, FORMAT_MD, FORMAT_RST, FORMAT_TXT]), 'output-style': ("Control output style; auto implies using Rich if available to produce rich output, " "with fallback to basic colored output", 'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES), From 81383e2dc9f1b612f9788e69ed746a6da51801d4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 29 Dec 2023 18:33:26 +0100 Subject: [PATCH 244/312] minor tweaks to 4.9.0 release notes --- RELEASE_NOTES | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 641a0d70d6..94b66db903 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -11,13 +11,14 @@ feature release - various enhancements, including: - allow tweaking of easyconfigs from different toolchains (#3669) + - add support for `--list-software --output-format=json` (#4152) - add `--module-cache-suffix` configuration setting to allow multiple (Lmod) caches (#4403) - various bug fixes, including: - deduplicate warnings & errors found in logs and add initial newline + tab in output (#4361) - fix support for Environment Modules as modules tool to pass unit tests with v4.2+ (#4369) - - Adapt module function check for Environment Modules v4+ (#4371) + - adapt module function check for Environment Modules v4+ (#4371) - only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9 (#4375) - - Use `-qopenmp` instead of `-fiopenmp` for OpenMP in Intel compilers (#4377) + - use `-qopenmp` instead of `-fiopenmp` for OpenMP in Intel compilers (#4377) - fix `LIBBLAS_MT` for FlexiBLAS, ensure `-lpthread` is included (#4379) - relax major version match regex in `find_related_easyconfigs` using for `--review-pr` (#4385) - eliminate duplicate multideps from generated module files (#4386) @@ -25,7 +26,7 @@ feature release - use source toolchain version when passing only `--try-toolchain` (#4395) - fix writing spider cache for Lmod >= 8.7.12 (#4402) - fix `--inject-checksums` when extension specifies patch file in tuple format (#4405) - - fix LooseVersion on Python 2.7 (#4408) + - fix `LooseVersion` when running with Python 2.7 (#4408) - use more recent easyblocks PR in `test_github_merge_pr` (#4414) - other changes: - extend test that checks build environment to recent `foss/2023a` toolchain (#4391) From 7bebf0ad2ff94c36c479aa89f8dee2de3b97952c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 30 Dec 2023 22:30:53 +0100 Subject: [PATCH 245/312] bump version to 4.9.1dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index e3d5feb017..d296c6112d 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -45,7 +45,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.9.0') +VERSION = LooseVersion('4.9.1.dev0') UNKNOWN = 'UNKNOWN' From fc27fe26a497b46fa19b7a565e001bd7187ac27a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 31 Dec 2023 15:37:13 +0100 Subject: [PATCH 246/312] use run_shell_cmd in scripts --- easybuild/scripts/findPythonDeps.py | 12 ++++++------ easybuild/scripts/mk_tmpl_easyblock_for.py | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index d6e496a048..31c990c468 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -48,7 +48,7 @@ def can_run(cmd, argument): return False -def run_cmd(arguments, action_desc, capture_stderr=True, **kwargs): +def run_shell_cmd(arguments, action_desc, capture_stderr=True, **kwargs): """Run the command and return the return code and output""" extra_args = kwargs or {} if sys.version_info[0] >= 3: @@ -66,7 +66,7 @@ def run_cmd(arguments, action_desc, capture_stderr=True, **kwargs): def run_in_venv(cmd, venv_path, action_desc): """Run the given command in the virtualenv at the given path""" cmd = 'source %s/bin/activate && %s' % (venv_path, cmd) - return run_cmd(cmd, action_desc, shell=True, executable='/bin/bash') + return run_shell_cmd(cmd, action_desc, shell=True, executable='/bin/bash') def get_dep_tree(package_spec, verbose): @@ -78,7 +78,7 @@ def get_dep_tree(package_spec, verbose): venv_dir = os.path.join(tmp_dir, 'venv') if verbose: print('Creating virtualenv at ' + venv_dir) - run_cmd(['virtualenv', '--system-site-packages', venv_dir], action_desc='create virtualenv') + run_shell_cmd(['virtualenv', '--system-site-packages', venv_dir], action_desc='create virtualenv') if verbose: print('Updating pip in virtualenv') run_in_venv('pip install --upgrade pip', venv_dir, action_desc='update pip') @@ -169,7 +169,7 @@ def print_deps(package, verbose): sys.exit(1) if args.verbose: print('Checking with EasyBuild for missing dependencies') - missing_dep_out = run_cmd(['eb', args.ec, '--missing'], + missing_dep_out = run_shell_cmd(['eb', args.ec, '--missing'], capture_stderr=False, action_desc='Get missing dependencies' ) @@ -189,7 +189,7 @@ def print_deps(package, verbose): os.chdir(tmp_dir) if args.verbose: print('Running EasyBuild to get build environment') - run_cmd(['eb', ec_arg, '--dump-env', '--force'], action_desc='Dump build environment') + run_shell_cmd(['eb', ec_arg, '--dump-env', '--force'], action_desc='Dump build environment') os.chdir(old_dir) cmd = "source %s/*.env && python %s '%s'" % (tmp_dir, sys.argv[0], args.package) @@ -197,7 +197,7 @@ def print_deps(package, verbose): cmd += ' --verbose' print('Restarting script in new build environment') - out = run_cmd(cmd, action_desc='Run in new environment', shell=True, executable='/bin/bash') + out = run_shell_cmd(cmd, action_desc='Run in new environment', shell=True, executable='/bin/bash') print(out) else: if not can_run('virtualenv', '--version'): diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py index c5c7c0110a..1cd63393d6 100755 --- a/easybuild/scripts/mk_tmpl_easyblock_for.py +++ b/easybuild/scripts/mk_tmpl_easyblock_for.py @@ -122,7 +122,7 @@ import easybuild.tools.toolchain as toolchain %(parent_import)s from easybuild.framework.easyconfig import CUSTOM, MANDATORY -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class %(class_name)s(%(parent)s): @@ -150,7 +150,7 @@ def configure_step(self): env.setvar('CUSTOM_ENV_VAR', 'foo') cmd = "configure command" - run_cmd(cmd, log_all=True, simple=True, log_ok=True) + run_shell_cmd(cmd) # complete configuration with configure_method of parent super(%(class_name)s, self).configure_step() @@ -167,20 +167,20 @@ def build_step(self): # enable parallel build par = self.cfg['parallel'] cmd = "build command --parallel %%d --compiler-family %%s" %% (par, comp_fam) - run_cmd(cmd, log_all=True, simple=True, log_ok=True) + run_shell_cmd(cmd) def test_step(self): \"\"\"Custom built-in test procedure for %(name)s.\"\"\" if self.cfg['runtest']: cmd = "test-command" - run_cmd(cmd, simple=True, log_all=True, log_ok=True) + run_shell_cmd(cmd) def install_step(self): \"\"\"Custom install procedure for %(name)s.\"\"\" cmd = "install command" - run_cmd(cmd, log_all=True, simple=True, log_ok=True) + run_shell_cmd(cmd) def sanity_check_step(self): \"\"\"Custom sanity check for %(name)s.\"\"\" From df371cb4eb1773bf5987973023d1675d4aded2c7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 1 Jan 2024 18:55:39 +0100 Subject: [PATCH 247/312] replace use of run_cmd with run_shell_cmd in modules.py --- easybuild/tools/modules.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 163764abb3..36368fe6e0 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1358,7 +1358,8 @@ def check_module_function(self, allow_mismatch=False, regex=None): out, ec = None, 1 else: cmd = "type _module_raw" - out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, force_in_dry_run=True, trace=False) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=False, hidden=True, output_file=False) + out, ec = res.output, res.exit_code if regex is None: regex = r".*%s" % os.path.basename(self.cmd) From bc76fb3cb8562f5be58f82e3a346c45ad87352e0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 1 Jan 2024 19:02:03 +0100 Subject: [PATCH 248/312] fix code style in findPythonDeps.py script --- easybuild/scripts/findPythonDeps.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index 31c990c468..1933bcf1df 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -170,9 +170,8 @@ def print_deps(package, verbose): if args.verbose: print('Checking with EasyBuild for missing dependencies') missing_dep_out = run_shell_cmd(['eb', args.ec, '--missing'], - capture_stderr=False, - action_desc='Get missing dependencies' - ) + capture_stderr=False, + action_desc='Get missing dependencies') excluded_dep = '(%s)' % os.path.basename(args.ec) missing_deps = [dep for dep in missing_dep_out.split('\n') if dep.startswith('*') and excluded_dep not in dep From 38f32bdebc4fa4a029b1d0d8650bfb86039f280c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 2 Jan 2024 20:15:11 +0100 Subject: [PATCH 249/312] implement support for 'stream_output' option in run_shell_cmd --- easybuild/tools/run.py | 52 ++++++++++++++++++++++++++++++++---------- test/framework/run.py | 34 +++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 12 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 1a03cbe08e..0bb0d3c9f4 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -182,7 +182,7 @@ def cache_aware_func(cmd, *args, **kwargs): @run_shell_cmd_cache def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, use_bash=True, - output_file=True, stream_output=False, asynchronous=False, with_hooks=True, + output_file=True, stream_output=None, asynchronous=False, with_hooks=True, qa_patterns=None, qa_wait_patterns=None): """ Run specified (interactive) shell command, and capture output + exit code. @@ -197,7 +197,7 @@ def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=N :param work_dir: working directory to run command in (current working directory if None) :param use_bash: execute command through bash shell (enabled by default) :param output_file: collect command output in temporary output file - :param stream_output: stream command output to stdout + :param stream_output: stream command output to stdout (auto-enabled with --logtostdout if None) :param asynchronous: run command asynchronously :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined) :param qa_patterns: list of 2-tuples with patterns for questions + corresponding answers @@ -222,7 +222,7 @@ def to_cmd_str(cmd): return cmd_str # temporarily raise a NotImplementedError until all options are implemented - if any((stream_output, asynchronous)): + if asynchronous: raise NotImplementedError if qa_patterns or qa_wait_patterns: @@ -234,6 +234,11 @@ def to_cmd_str(cmd): cmd_str = to_cmd_str(cmd) cmd_name = os.path.basename(cmd_str.split(' ')[0]) + # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely + if stream_output is None and build_option('logtostdout'): + _log.info(f"Auto-enabling streaming output of '{cmd_str}' command because logging to stdout is enabled") + stream_output = True + # temporary output file(s) for command output if output_file: toptmpdir = os.path.join(tempfile.gettempdir(), 'run-shell-cmd-output') @@ -264,9 +269,8 @@ def to_cmd_str(cmd): if not hidden: cmd_trace_msg(cmd_str, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp) - if stdin: - # 'input' value fed to subprocess.run must be a byte sequence - stdin = stdin.encode() + if stream_output: + print_msg(f"(streaming) output for command '{cmd_str}':") # use bash as shell instead of the default /bin/sh used by subprocess.run # (which could be dash instead of bash, like on Ubuntu, see https://wiki.ubuntu.com/DashAsBinSh) @@ -276,8 +280,6 @@ def to_cmd_str(cmd): else: executable, shell = None, False - stderr = subprocess.PIPE if split_stderr else subprocess.STDOUT - if with_hooks: hooks = load_hooks(build_option('hooks')) hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'work_dir': work_dir}) @@ -286,13 +288,39 @@ def to_cmd_str(cmd): cmd_str = to_cmd_str(cmd) _log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd) + stderr = subprocess.PIPE if split_stderr else subprocess.STDOUT + _log.info(f"Running command '{cmd_str}' in {work_dir}") - proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=stderr, check=False, - cwd=work_dir, env=env, input=stdin, shell=shell, executable=executable) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr, stdin=subprocess.PIPE, + cwd=work_dir, env=env, shell=shell, executable=executable) + + # 'input' value fed to subprocess.run must be a byte sequence + if stdin: + stdin = stdin.encode() + + if stream_output: + if stdin: + proc.stdin.write(stdin) + + exit_code = None + stdout, stderr = b'', b'' + + while exit_code is None: + exit_code = proc.poll() + + # use small read size (128 bytes) when streaming output, to make it stream more fluently + # -1 means reading until EOF + read_size = 128 if exit_code is None else -1 + + stdout += proc.stdout.read(read_size) + if split_stderr: + stderr += proc.stderr.read(read_size) + else: + (stdout, stderr) = proc.communicate(input=stdin) # return output as a regular string rather than a byte sequence (and non-UTF-8 characters get stripped out) - output = proc.stdout.decode('utf-8', 'ignore') - stderr = proc.stderr.decode('utf-8', 'ignore') if split_stderr else None + output = stdout.decode('utf-8', 'ignore') + stderr = stderr.decode('utf-8', 'ignore') if split_stderr else None # store command output to temporary file(s) if output_file: diff --git a/test/framework/run.py b/test/framework/run.py index 4347dcdeeb..db74940aec 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1107,6 +1107,40 @@ def test_run_cmd_stream(self): for line in expected: self.assertIn(line, stdout) + def test_run_shell_cmd_stream(self): + """Test use of run_shell_cmd with streaming output.""" + self.mock_stdout(True) + self.mock_stderr(True) + cmd = '; '.join([ + "echo hello there", + "sleep 1", + "echo testing command that produces a fair amount of output", + "sleep 1", + "echo more than 128 bytes which means a whole bunch of characters...", + "sleep 1", + "echo more than 128 characters in fact, which is quite a bit when you think of it", + ]) + res = run_shell_cmd(cmd, stream_output=True) + stdout = self.get_stdout() + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + + expected_output = '\n'.join([ + "hello there", + "testing command that produces a fair amount of output", + "more than 128 bytes which means a whole bunch of characters...", + "more than 128 characters in fact, which is quite a bit when you think of it", + '', + ]) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, expected_output) + + self.assertEqual(stderr, '') + expected = ("== (streaming) output for command 'echo hello" + '\n' + expected_output).split('\n') + for line in expected: + self.assertIn(line, stdout) + def test_run_cmd_async(self): """Test asynchronously running of a shell command via run_cmd + complete_cmd.""" From c98825dbdfa874179ec1c55b210cc80105793983 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 2 Jan 2024 20:15:19 +0100 Subject: [PATCH 250/312] disable stream_output when running shell commands in systemtools --- easybuild/tools/systemtools.py | 39 +++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 32caf0b4a0..a2dfb08185 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -273,7 +273,8 @@ def get_avail_core_count(): core_cnt = int(sum(sched_getaffinity())) else: # BSD-type systems - res = run_shell_cmd('sysctl -n hw.ncpu', in_dry_run=True, hidden=True, with_hooks=False, output_file=False) + res = run_shell_cmd('sysctl -n hw.ncpu', in_dry_run=True, hidden=True, with_hooks=False, + output_file=False, stream_output=False) try: if int(res.output) > 0: core_cnt = int(res.output) @@ -310,7 +311,7 @@ def get_total_memory(): elif os_type == DARWIN: cmd = "sysctl -n hw.memsize" _log.debug("Trying to determine total memory size on Darwin via cmd '%s'", cmd) - res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False) + res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False, stream_output=False) if res.exit_code == 0: memtotal = int(res.output.strip()) // (1024**2) @@ -392,7 +393,8 @@ def get_cpu_vendor(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.vendor" - res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, output_file=False) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, + output_file=False, stream_output=False) out = res.output.strip() if res.exit_code == 0 and out in VENDOR_IDS: vendor = VENDOR_IDS[out] @@ -400,7 +402,7 @@ def get_cpu_vendor(): else: cmd = "sysctl -n machdep.cpu.brand_string" res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, - output_file=False) + output_file=False, stream_output=False) out = res.output.strip().split(' ')[0] if res.exit_code == 0 and out in CPU_VENDORS: vendor = out @@ -503,7 +505,7 @@ def get_cpu_model(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.brand_string" - res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False) + res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False, stream_output=False) if res.exit_code == 0: model = res.output.strip() _log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model)) @@ -548,7 +550,7 @@ def get_cpu_speed(): elif os_type == DARWIN: cmd = "sysctl -n hw.cpufrequency_max" _log.debug("Trying to determine CPU frequency on Darwin via cmd '%s'" % cmd) - res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False) + res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False, stream_output=False) out = res.output.strip() cpu_freq = None if res.exit_code == 0 and out: @@ -597,7 +599,7 @@ def get_cpu_features(): cmd = "sysctl -n machdep.cpu.%s" % feature_set _log.debug("Trying to determine CPU features on Darwin via cmd '%s'", cmd) res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, fail_on_error=False, with_hooks=False, - output_file=False) + output_file=False, stream_output=False) if res.exit_code == 0: cpu_feat.extend(res.output.strip().lower().split()) @@ -625,7 +627,7 @@ def get_gpu_info(): cmd = "nvidia-smi --query-gpu=gpu_name,driver_version --format=csv,noheader" _log.debug("Trying to determine NVIDIA GPU info on Linux via cmd '%s'", cmd) res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, - output_file=False) + output_file=False, stream_output=False) if res.exit_code == 0: for line in res.output.strip().split('\n'): nvidia_gpu_info = gpu_info.setdefault('NVIDIA', {}) @@ -644,14 +646,14 @@ def get_gpu_info(): cmd = "rocm-smi --showdriverversion --csv" _log.debug("Trying to determine AMD GPU driver on Linux via cmd '%s'", cmd) res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, - output_file=False) + output_file=False, stream_output=False) if res.exit_code == 0: amd_driver = res.output.strip().split('\n')[1].split(',')[1] cmd = "rocm-smi --showproductname --csv" _log.debug("Trying to determine AMD GPU info on Linux via cmd '%s'", cmd) res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, - output_file=False) + output_file=False, stream_output=False) if res.exit_code == 0: for line in res.output.strip().split('\n')[1:]: amd_card_series = line.split(',')[1] @@ -870,7 +872,8 @@ def check_os_dependency(dep): pkg_cmd_flag.get(pkg_cmd), dep, ]) - res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, output_file=False) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, + output_file=False, stream_output=False) found = res.exit_code == 0 if found: break @@ -882,7 +885,8 @@ def check_os_dependency(dep): # try locate if it's available if not found and which('locate'): cmd = 'locate -c --regexp "/%s$"' % dep - res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, output_file=False) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, + output_file=False, stream_output=False) try: found = (res.exit_code == 0 and int(res.output.strip()) > 0) except ValueError: @@ -898,7 +902,7 @@ def get_tool_version(tool, version_option='--version', ignore_ec=False): Output is returned as a single-line string (newlines are replaced by '; '). """ res = run_shell_cmd(' '.join([tool, version_option]), fail_on_error=False, in_dry_run=True, - hidden=True, with_hooks=False, output_file=False) + hidden=True, with_hooks=False, output_file=False, stream_output=False) if not ignore_ec and res.exit_code: _log.warning("Failed to determine version of %s using '%s %s': %s" % (tool, tool, version_option, res.output)) return UNKNOWN @@ -910,7 +914,8 @@ def get_gcc_version(): """ Process `gcc --version` and return the GCC version. """ - res = run_shell_cmd('gcc --version', fail_on_error=False, in_dry_run=True, hidden=True, output_file=False) + res = run_shell_cmd('gcc --version', fail_on_error=False, in_dry_run=True, hidden=True, + output_file=False, stream_output=False) gcc_ver = None if res.exit_code: _log.warning("Failed to determine the version of GCC: %s", res.output) @@ -966,7 +971,7 @@ def get_linked_libs_raw(path): or None for other types of files. """ - res = run_shell_cmd("file %s" % path, fail_on_error=False, hidden=True, output_file=False) + res = run_shell_cmd("file %s" % path, fail_on_error=False, hidden=True, output_file=False, stream_output=False) if res.exit_code: fail_msg = "Failed to run 'file %s': %s" % (path, res.output) _log.warning(fail_msg) @@ -1001,7 +1006,7 @@ def get_linked_libs_raw(path): # take into account that 'ldd' may fail for strange reasons, # like printing 'not a dynamic executable' when not enough memory is available # (see also https://bugzilla.redhat.com/show_bug.cgi?id=1817111) - res = run_shell_cmd(linked_libs_cmd, fail_on_error=False, hidden=True, output_file=False) + res = run_shell_cmd(linked_libs_cmd, fail_on_error=False, hidden=True, output_file=False, stream_output=False) if res.exit_code == 0: linked_libs_out = res.output else: @@ -1183,7 +1188,7 @@ def get_default_parallelism(): # No cache -> Calculate value from current system values par = get_avail_core_count() # determine max user processes via ulimit -u - res = run_shell_cmd("ulimit -u", in_dry_run=True, hidden=True, output_file=False) + res = run_shell_cmd("ulimit -u", in_dry_run=True, hidden=True, output_file=False, stream_output=False) try: if res.output.startswith("unlimited"): maxuserproc = 2 ** 32 - 1 From 2d5db91d92c97ae3ee7ed22a50cb20f3eb56e0c6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Jan 2024 09:04:59 +0100 Subject: [PATCH 251/312] add comment to explain why hidden bzip2 module is not included in "avail" output for Tcl-based environment modules tool --- test/framework/modules.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/modules.py b/test/framework/modules.py index 9151527aac..2a05395b7c 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -220,6 +220,7 @@ def test_avail(self): self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) elif (isinstance(self.modtool, EnvironmentModules) and StrictVersion(self.modtool.version) >= StrictVersion('4.6.0')): + # bzip2/.1.0.6 is not there, since that's a module file in Lua syntax self.assertEqual(len(ms), TEST_MODULES_COUNT + 2) self.assertIn('toy/.0.0-deps', ms) self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) From 7f1f7eac5dbcbe32f1bffaa4839bea28b3158a7e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Jan 2024 12:14:14 +0100 Subject: [PATCH 252/312] switch to run_shell_cmd in easybuild/tools/container --- easybuild/tools/containers/apptainer.py | 18 ++++++++-------- easybuild/tools/containers/docker.py | 4 ++-- easybuild/tools/containers/singularity.py | 18 ++++++++-------- easybuild/tools/containers/utils.py | 25 +++++++++++++---------- 4 files changed, 34 insertions(+), 31 deletions(-) diff --git a/easybuild/tools/containers/apptainer.py b/easybuild/tools/containers/apptainer.py index 67a6db5bc3..8a294ce1ad 100644 --- a/easybuild/tools/containers/apptainer.py +++ b/easybuild/tools/containers/apptainer.py @@ -35,7 +35,7 @@ from easybuild.tools.config import CONT_IMAGE_FORMAT_SIF, CONT_IMAGE_FORMAT_SQUASHFS from easybuild.tools.config import build_option, container_path from easybuild.tools.filetools import remove_file, which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class ApptainerContainer(SingularityContainer): @@ -48,15 +48,15 @@ class ApptainerContainer(SingularityContainer): def apptainer_version(): """Get Apptainer version.""" version_cmd = "apptainer --version" - out, ec = run_cmd(version_cmd, simple=False, trace=False, force_in_dry_run=True) - if ec: - raise EasyBuildError("Error running '%s': %s for tool {1} with output: {2}" % (version_cmd, out)) + res = run_shell_cmd(version_cmd, hidden=True, in_dry_run=True) + if res.exit_code: + raise EasyBuildError(f"Error running '{version_cmd}': {res.output}") - res = re.search(r"\d+\.\d+(\.\d+)?", out.strip()) - if not res: - raise EasyBuildError("Error parsing Apptainer version: %s" % out) + regex_res = re.search(r"\d+\.\d+(\.\d+)?", res.output.strip()) + if not regex_res: + raise EasyBuildError(f"Error parsing Apptainer version: {res.output}") - return res.group(0) + return regex_res.group(0) def build_image(self, recipe_path): """Build container image by calling out to 'sudo apptainer build'.""" @@ -108,5 +108,5 @@ def build_image(self, recipe_path): cmd = ' '.join(['sudo', cmd_env, apptainer, 'build', cmd_opts, img_path, recipe_path]) print_msg("Running '%s', you may need to enter your 'sudo' password..." % cmd) - run_cmd(cmd, stream_output=True) + run_shell_cmd(cmd, stream_output=True) print_msg("Apptainer image created at %s" % img_path, log=self.log) diff --git a/easybuild/tools/containers/docker.py b/easybuild/tools/containers/docker.py index aa37a90873..5a1597d634 100644 --- a/easybuild/tools/containers/docker.py +++ b/easybuild/tools/containers/docker.py @@ -36,7 +36,7 @@ from easybuild.tools.containers.utils import det_os_deps from easybuild.tools.filetools import remove_dir from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd DOCKER_TMPL_HEADER = """\ @@ -164,7 +164,7 @@ def build_image(self, dockerfile): docker_cmd = ' '.join(['sudo', 'docker', 'build', '-f', dockerfile, '-t', container_name, '.']) print_msg("Running '%s', you may need to enter your 'sudo' password..." % docker_cmd) - run_cmd(docker_cmd, path=tempdir, stream_output=True) + run_shell_cmd(docker_cmd, work_dir=tempdir, stream_output=True) print_msg("Docker image created at %s" % container_name, log=self.log) remove_dir(tempdir) diff --git a/easybuild/tools/containers/singularity.py b/easybuild/tools/containers/singularity.py index 41ec9829c7..638a985ae5 100644 --- a/easybuild/tools/containers/singularity.py +++ b/easybuild/tools/containers/singularity.py @@ -40,7 +40,7 @@ from easybuild.tools.config import build_option, container_path from easybuild.tools.containers.base import ContainerGenerator from easybuild.tools.filetools import read_file, remove_file, which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd ARCH = 'arch' # Arch Linux @@ -162,15 +162,15 @@ class SingularityContainer(ContainerGenerator): def singularity_version(): """Get Singularity version.""" version_cmd = "singularity --version" - out, ec = run_cmd(version_cmd, simple=False, trace=False, force_in_dry_run=True) - if ec: - raise EasyBuildError("Error running '%s': %s for tool {1} with output: {2}" % (version_cmd, out)) + res = run_shell_cmd(version_cmd, hidden=True, in_dry_run=True) + if res.exit_code: + raise EasyBuildError(f"Error running '{version_cmd}': {res.output}") - res = re.search(r"\d+\.\d+(\.\d+)?", out.strip()) - if not res: - raise EasyBuildError("Error parsing Singularity version: %s" % out) + regex_res = re.search(r"\d+\.\d+(\.\d+)?", res.output.strip()) + if not regex_res: + raise EasyBuildError(f"Error parsing Singularity version: {res.output}") - return res.group(0) + return regex_res.group(0) def resolve_template(self): """Return template container recipe.""" @@ -403,5 +403,5 @@ def build_image(self, recipe_path): cmd = ' '.join(['sudo', cmd_env, singularity, 'build', cmd_opts, img_path, recipe_path]) print_msg("Running '%s', you may need to enter your 'sudo' password..." % cmd) - run_cmd(cmd, stream_output=True) + run_shell_cmd(cmd, stream_output=True) print_msg("Singularity image created at %s" % img_path, log=self.log) diff --git a/easybuild/tools/containers/utils.py b/easybuild/tools/containers/utils.py index e01a117427..d543ca77fc 100644 --- a/easybuild/tools/containers/utils.py +++ b/easybuild/tools/containers/utils.py @@ -36,7 +36,7 @@ from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.filetools import which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd def det_os_deps(easyconfigs): @@ -70,20 +70,23 @@ def check_tool(tool_name, min_tool_version=None): if not tool_path: return False - print_msg("{0} tool found at {1}".format(tool_name, tool_path)) + print_msg(f"{tool_name} tool found at {tool_path}") if not min_tool_version: return True - version_cmd = "{0} --version".format(tool_name) - out, ec = run_cmd(version_cmd, simple=False, trace=False, force_in_dry_run=True) - if ec: - raise EasyBuildError("Error running '{0}' for tool {1} with output: {2}".format(version_cmd, tool_name, out)) - res = re.search(r"\d+\.\d+(\.\d+)?", out.strip()) - if not res: - raise EasyBuildError("Error parsing version for tool {0}".format(tool_name)) - tool_version = res.group(0) + version_cmd = f"{tool_name} --version" + res = run_shell_cmd(version_cmd, hidden=True, in_dry_run=True) + if res.exit_code: + raise EasyBuildError(f"Error running '{version_cmd}' for tool {tool_name} with output: {res.output}") + + regex_res = re.search(r"\d+\.\d+(\.\d+)?", res.output.strip()) + if not regex_res: + raise EasyBuildError(f"Error parsing version for tool {tool_name}") + + tool_version = regex_res.group(0) version_ok = LooseVersion(str(min_tool_version)) <= LooseVersion(tool_version) if version_ok: - print_msg("{0} version '{1}' is {2} or higher ... OK".format(tool_name, tool_version, min_tool_version)) + print_msg(f"{tool_name} version '{tool_version}' is {min_tool_version} or higher ... OK") + return version_ok From b05b07ddddfb43861015ab2eb685d73260ed8272 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 3 Jan 2024 14:16:58 +0000 Subject: [PATCH 253/312] bump minimum required Lmod to 8.0.0 --- easybuild/tools/modules.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 36368fe6e0..7e106ae1d2 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1406,9 +1406,8 @@ class Lmod(ModulesTool): NAME = "Lmod" COMMAND = 'lmod' COMMAND_ENVIRONMENT = 'LMOD_CMD' - REQ_VERSION = '6.5.1' - DEPR_VERSION = '7.0.0' - REQ_VERSION_DEPENDS_ON = '7.6.1' + REQ_VERSION = '8.0.0' + DEPR_VERSION = '8.0.0' VERSION_REGEXP = r"^Modules\s+based\s+on\s+Lua:\s+Version\s+(?P\d\S*)\s" SHOW_HIDDEN_OPTION = '--show-hidden' @@ -1427,7 +1426,7 @@ def __init__(self, *args, **kwargs): super(Lmod, self).__init__(*args, **kwargs) version = StrictVersion(self.version) - self.supports_depends_on = version >= self.REQ_VERSION_DEPENDS_ON + self.supports_depends_on = True # See https://lmod.readthedocs.io/en/latest/125_personal_spider_cache.html if version >= '8.7.12': self.USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.cache', 'lmod') @@ -1586,13 +1585,9 @@ def module_wrapper_exists(self, mod_name): Determine whether a module wrapper with specified name exists. First check for wrapper defined in .modulerc.lua, fall back to also checking .modulerc (Tcl syntax). """ - res = None - - # first consider .modulerc.lua with Lmod 7.8 (or newer) - if StrictVersion(self.version) >= StrictVersion('7.8'): - mod_wrapper_regex_template = r'^module_version\("(?P.*)", "%s"\)$' - res = super(Lmod, self).module_wrapper_exists(mod_name, modulerc_fn='.modulerc.lua', - mod_wrapper_regex_template=mod_wrapper_regex_template) + mod_wrapper_regex_template = r'^module_version\("(?P.*)", "%s"\)$' + res = super(Lmod, self).module_wrapper_exists(mod_name, modulerc_fn='.modulerc.lua', + mod_wrapper_regex_template=mod_wrapper_regex_template) # fall back to checking for .modulerc in Tcl syntax if res is None: From 5cdea4615623479f68d2f34169a971c5cef26e25 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 3 Jan 2024 14:35:39 +0000 Subject: [PATCH 254/312] remove Lmod 7 testing --- .github/workflows/unit_tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 08408aea75..225b3f85c6 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -13,7 +13,6 @@ jobs: setup: runs-on: ubuntu-20.04 outputs: - lmod7: Lmod-7.8.22 lmod8: Lmod-8.7.6 modulesTcl: modules-tcl-1.147 modules3: modules-3.2.10 @@ -29,7 +28,6 @@ jobs: modules_tool: # use variables defined by 'setup' job above, see also # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context - - ${{needs.setup.outputs.lmod7}} - ${{needs.setup.outputs.lmod8}} - ${{needs.setup.outputs.modulesTcl}} - ${{needs.setup.outputs.modules3}} From d7f2fafcfca7df863b58a1ef1ad4421138d5a557 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Jan 2024 16:03:53 +0100 Subject: [PATCH 255/312] tweak warning/error message when .mod files are found --- easybuild/framework/easyblock.py | 6 ++++-- test/framework/toy_build.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 954bb3212e..4fba08ccab 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3312,7 +3312,7 @@ def sanity_check_mod_files(self): fail_msg = None if mod_files: - fail_msg = ".mod files (%s) found in the installation." % ', '.join(mod_files) + fail_msg = f"One or more .mod files found in {self.installdir}: " + ', '.join(mod_files) return fail_msg @@ -3638,7 +3638,9 @@ def xs2str(xs): self.log.warning("Check for required/banned linked shared libraries failed!") self.sanity_check_fail_msgs.append(linked_shared_lib_fails) - if self.toolchain.name in ['GCCcore'] and not self.cfg['skip_mod_files_sanity_check']: + # software installed with GCCcore toolchain should not have Fortran module files (.mod), + # unless that's explicitly allowed + if self.toolchain.name in ('GCCcore',) and not self.cfg['skip_mod_files_sanity_check']: mod_files_found_msg = self.sanity_check_mod_files() if mod_files_found_msg: if build_option('fail_on_mod_files_gcccore'): diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 9bf85607fa..909c3b6664 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3979,11 +3979,11 @@ def test_toy_mod_files(self): stderr = self.get_stderr() self.mock_stdout(False) self.mock_stderr(False) - pattern = r"WARNING: .mod files (.*) found in the installation." + pattern = r"WARNING: One or more \.mod files found in .*/software/toy/0.0-GCCcore-6.2.0: .*/lib64/file.mod" self.assertRegex(stderr.strip(), pattern) args += ['--fail-on-mod-files-gcccore'] - pattern = r"Sanity check failed: .mod files (.*) found in the installation." + pattern = r"Sanity check failed: One or more \.mod files found in .*/toy/0.0-GCCcore-6.2.0: .*/lib/file.mod" self.assertErrorRegex(EasyBuildError, pattern, self.run_test_toy_build_with_output, ec_file=test_ec, extra_args=args, verify=False, fails=True, verbose=False, raise_error=True) From 8b4a889a2acc8bbc0be04d018da2e1e1c9ae2265 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Jan 2024 21:02:05 +0100 Subject: [PATCH 256/312] switch to run_shell_cmd in toy_build test module --- .../easybuild/easyblocks/t/toy_buggy.py | 2 +- test/framework/toy_build.py | 58 ++++++++++--------- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py index 3695744630..43d7433265 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py @@ -41,7 +41,7 @@ def configure_step(self): def build_step(self): """Build toy.""" # note: import is (purposely) missing, so this will go down hard - run_cmd('gcc toy.c -o toy') # noqa + run_shell_cmd('gcc toy.c -o toy') # noqa def install_step(self): """Install toy.""" diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 909c3b6664..d4c7988534 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -58,7 +58,7 @@ from easybuild.tools.filetools import read_file, remove_dir, remove_file, which, write_file from easybuild.tools.module_generator import ModuleGeneratorTcl from easybuild.tools.modules import Lmod -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.utilities import nub from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.version import VERSION as EASYBUILD_VERSION @@ -374,7 +374,7 @@ def test_toy_buggy_easyblock(self): 'verify': False, 'verbose': False, } - err_regex = r"name 'run_cmd' is not defined" + err_regex = r"name 'run_shell_cmd' is not defined" self.assertErrorRegex(NameError, err_regex, self.run_test_toy_build_with_output, **kwargs) def test_toy_build_formatv2(self): @@ -742,9 +742,9 @@ def test_toy_group_check(self): # figure out a group that we're a member of to use in the test with self.mocked_stdout_stderr(): - out, ec = run_cmd('groups', simple=False) - self.assertEqual(ec, 0, "Failed to select group to use in test") - group_name = out.split(' ')[0].strip() + res = run_shell_cmd('groups') + self.assertEqual(res.exit_code, 0, "Failed to select group to use in test") + group_name = res.output.split(' ')[0].strip() toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') @@ -1284,8 +1284,8 @@ def test_toy_extension_patches_postinstallcmds(self): # make sure that patches were actually applied (without them the message producded by 'bar' is different) bar_bin = os.path.join(installdir, 'bin', 'bar') with self.mocked_stdout_stderr(): - out, _ = run_cmd(bar_bin) - self.assertEqual(out, "I'm a bar, and very very proud of it.\n") + res = run_shell_cmd(bar_bin) + self.assertEqual(res.output, "I'm a bar, and very very proud of it.\n") # verify that post-install command for 'bar' extension was executed fn = 'created-via-postinstallcmds.txt' @@ -2819,17 +2819,19 @@ def test_toy_filter_rpath_sanity_libs(self): libtoy_libdir = os.path.join(self.test_installpath, 'software', 'libtoy', '0.0', 'lib') toyapp_bin = os.path.join(self.test_installpath, 'software', 'toy-app', '0.0', 'bin', 'toy-app') - rpath_regex = re.compile(r"RPATH.*%s" % libtoy_libdir, re.M) + rpath_regex = re.compile(r"RPATH.*" + libtoy_libdir, re.M) with self.mocked_stdout_stderr(): - out, ec = run_cmd("readelf -d %s" % toyapp_bin, simple=False) - self.assertTrue(rpath_regex.search(out), "Pattern '%s' should be found in: %s" % (rpath_regex.pattern, out)) + res = run_shell_cmd(f"readelf -d {toyapp_bin}") + self.assertTrue(rpath_regex.search(res.output), + f"Pattern '{rpath_regex.pattern}' should be found in: {res.output}") with self.mocked_stdout_stderr(): - out, ec = run_cmd("ldd %s" % toyapp_bin, simple=False) + res = run_shell_cmd(f"ldd {toyapp_bin}") + out = res.output libtoy_regex = re.compile(r"libtoy.so => /.*/libtoy.so", re.M) notfound = re.compile(r"libtoy\.so\s*=>\s*not found", re.M) - self.assertTrue(libtoy_regex.search(out), "Pattern '%s' should be found in: %s" % (libtoy_regex.pattern, out)) - self.assertFalse(notfound.search(out), "Pattern '%s' should not be found in: %s" % (notfound.pattern, out)) + self.assertTrue(libtoy_regex.search(out), f"Pattern '{libtoy_regex.pattern}' should be found in: {out}") + self.assertFalse(notfound.search(out), f"Pattern '{notfound.pattern}' should not be found in: {out}") # test sanity error when --rpath-filter is used to filter a required library # In this test, libtoy.so will be linked, but not RPATH-ed due to the --rpath-filter @@ -2848,16 +2850,16 @@ def test_toy_filter_rpath_sanity_libs(self): self._test_toy_build(ec_file=toy_ec, name='toy-app', extra_args=args, raise_error=True) with self.mocked_stdout_stderr(): - out, ec = run_cmd("readelf -d %s" % toyapp_bin, simple=False) - self.assertFalse(rpath_regex.search(out), - "Pattern '%s' should not be found in: %s" % (rpath_regex.pattern, out)) + res = run_shell_cmd(f"readelf -d {toyapp_bin}") + self.assertFalse(rpath_regex.search(res.output), + f"Pattern '{rpath_regex.pattern}' should not be found in: {res.output}") with self.mocked_stdout_stderr(): - out, ec = run_cmd("ldd %s" % toyapp_bin, simple=False) - self.assertFalse(libtoy_regex.search(out), - "Pattern '%s' should not be found in: %s" % (libtoy_regex.pattern, out)) - self.assertTrue(notfound.search(out), - "Pattern '%s' should be found in: %s" % (notfound.pattern, out)) + res = run_shell_cmd(f"ldd {toyapp_bin}") + self.assertFalse(libtoy_regex.search(res.output), + f"Pattern '{libtoy_regex.pattern}' should not be found in: {res.output}") + self.assertTrue(notfound.search(res.output), + f"Pattern '{notfound.pattern}' should be found in: {res.output}") # test again with list of library names passed to --filter-rpath-sanity-libs args = ['--rpath', '--rpath-filter=.*libtoy.*', '--filter-rpath-sanity-libs=libfoo.so,libtoy.so,libbar.so'] @@ -2865,16 +2867,16 @@ def test_toy_filter_rpath_sanity_libs(self): self._test_toy_build(ec_file=toy_ec, name='toy-app', extra_args=args, raise_error=True) with self.mocked_stdout_stderr(): - out, ec = run_cmd("readelf -d %s" % toyapp_bin, simple=False) + res = run_shell_cmd(f"readelf -d {toyapp_bin}") self.assertFalse(rpath_regex.search(out), - "Pattern '%s' should not be found in: %s" % (rpath_regex.pattern, out)) + f"Pattern '{rpath_regex.pattern}' should not be found in: {res.output}") with self.mocked_stdout_stderr(): - out, ec = run_cmd("ldd %s" % toyapp_bin, simple=False) - self.assertFalse(libtoy_regex.search(out), - "Pattern '%s' should not be found in: %s" % (libtoy_regex.pattern, out)) - self.assertTrue(notfound.search(out), - "Pattern '%s' should be found in: %s" % (notfound.pattern, out)) + res = run_shell_cmd(f"ldd {toyapp_bin}") + self.assertFalse(libtoy_regex.search(res.output), + f"Pattern '{libtoy_regex.pattern}' should not be found in: {res.output}") + self.assertTrue(notfound.search(res.output), + f"Pattern '{notfound.pattern}' should be found in: {res.output}") def test_toy_modaltsoftname(self): """Build two dependent toys as in test_toy_toy but using modaltsoftname""" From 84b8e5d71de59e1198a4dfb654f8cf2ef83a13df Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Jan 2024 21:55:42 +0100 Subject: [PATCH 257/312] switch to run_shell_cmd in toolchain test module --- test/framework/toolchain.py | 133 ++++++++++++++++++------------------ 1 file changed, 67 insertions(+), 66 deletions(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index aa090bda94..1c8c1771a6 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -50,7 +50,7 @@ from easybuild.tools.environment import setvar from easybuild.tools.filetools import adjust_permissions, copy_dir, find_eb_script, mkdir from easybuild.tools.filetools import read_file, symlink, write_file, which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.toolchain.mpi import get_mpi_cmd_template from easybuild.tools.toolchain.toolchain import env_vars_external_module @@ -2366,8 +2366,8 @@ def test_rpath_args_script(self): # simplest possible compiler command with self.mocked_stdout_stderr(): - out, ec = run_cmd("%s gcc '' '%s' -c foo.c" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(f"{script} gcc '' '{rpath_inc}' -c foo.c") + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2378,12 +2378,12 @@ def test_rpath_args_script(self): "'-c'", "'foo.c'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # linker command, --enable-new-dtags should be replaced with --disable-new-dtags with self.mocked_stdout_stderr(): - out, ec = run_cmd("%s ld '' '%s' --enable-new-dtags foo.o" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(f"{script} ld '' '{rpath_inc}' --enable-new-dtags foo.o") + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-rpath=%s/lib'" % self.test_prefix, "'-rpath=%s/lib64'" % self.test_prefix, @@ -2394,12 +2394,12 @@ def test_rpath_args_script(self): "'--disable-new-dtags'", "'foo.o'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # compiler command, -Wl,--enable-new-dtags should be replaced with -Wl,--disable-new-dtags with self.mocked_stdout_stderr(): - out, ec = run_cmd("%s gcc '' '%s' -Wl,--enable-new-dtags foo.c" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(f"{script} gcc '' '{rpath_inc}' -Wl,--enable-new-dtags foo.c") + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2410,12 +2410,12 @@ def test_rpath_args_script(self): "'-Wl,--disable-new-dtags'", "'foo.c'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # test passing no arguments with self.mocked_stdout_stderr(): - out, ec = run_cmd("%s gcc '' '%s'" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(f"{script} gcc '' '{rpath_inc}'") + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2424,12 +2424,12 @@ def test_rpath_args_script(self): "'-Wl,-rpath=$ORIGIN/../lib64'", "'-Wl,--disable-new-dtags'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # test passing a single empty argument with self.mocked_stdout_stderr(): - out, ec = run_cmd("%s ld.gold '' '%s' ''" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(f"{script} ld.gold '' '{rpath_inc}' ''") + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-rpath=%s/lib'" % self.test_prefix, "'-rpath=%s/lib64'" % self.test_prefix, @@ -2439,13 +2439,13 @@ def test_rpath_args_script(self): "'--disable-new-dtags'", "''", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # single -L argument, but non-existing path => not used in RPATH, but -L option is retained - cmd = "%s gcc '' '%s' foo.c -L%s/foo -lfoo" % (script, rpath_inc, self.test_prefix) + cmd = f"{script} gcc '' '{rpath_inc}' foo.c -L{self.test_prefix}/foo -lfoo" with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2457,13 +2457,13 @@ def test_rpath_args_script(self): "'-L%s/foo'" % self.test_prefix, "'-lfoo'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # single -L argument again, with existing path mkdir(os.path.join(self.test_prefix, 'foo')) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2476,12 +2476,12 @@ def test_rpath_args_script(self): "'-L%s/foo'" % self.test_prefix, "'-lfoo'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # relative paths passed to -L are *not* RPATH'ed in with self.mocked_stdout_stderr(): - out, ec = run_cmd("%s gcc '' '%s' foo.c -L../lib -lfoo" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(f"{script} gcc '' '{rpath_inc}' foo.c -L../lib -lfoo") + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2493,13 +2493,13 @@ def test_rpath_args_script(self): "'-L../lib'", "'-lfoo'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # single -L argument, with value separated by a space - cmd = "%s gcc '' '%s' foo.c -L %s/foo -lfoo" % (script, rpath_inc, self.test_prefix) + cmd = f"{script} gcc '' '{rpath_inc}' foo.c -L {self.test_prefix}/foo -lfoo" with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2512,7 +2512,7 @@ def test_rpath_args_script(self): "'-L%s/foo'" % self.test_prefix, "'-lfoo'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) mkdir(os.path.join(self.test_prefix, 'bar')) mkdir(os.path.join(self.test_prefix, 'lib64')) @@ -2534,8 +2534,8 @@ def test_rpath_args_script(self): '-L%s/bar' % self.test_prefix, ]) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-rpath=%s/lib'" % self.test_prefix, "'-rpath=%s/lib64'" % self.test_prefix, @@ -2556,7 +2556,7 @@ def test_rpath_args_script(self): "'-L/usr/lib'", "'-L%s/bar'" % self.test_prefix, ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # test specifying of custom rpath filter cmd = ' '.join([ @@ -2572,8 +2572,8 @@ def test_rpath_args_script(self): '-lbar', ]) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-rpath=%s/lib'" % self.test_prefix, "'-rpath=%s/lib64'" % self.test_prefix, @@ -2589,7 +2589,7 @@ def test_rpath_args_script(self): "'-L/bar'", "'-lbar'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # slightly trimmed down real-life example (compilation of XZ) for subdir in ['icc/lib/intel64', 'imkl/lib', 'imkl/mkl/lib/intel64', 'gettext/lib']: @@ -2612,8 +2612,8 @@ def test_rpath_args_script(self): '-Wl,/example/software/XZ/5.2.2-intel-2016b/lib', ]) with self.mocked_stdout_stderr(): - out, ec = run_cmd("%s icc '' '%s' %s" % (script, rpath_inc, args), simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(f"{script} icc '' '{rpath_inc}' {args}") + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2640,7 +2640,7 @@ def test_rpath_args_script(self): "'-Wl,-rpath'", "'-Wl,/example/software/XZ/5.2.2-intel-2016b/lib'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # trimmed down real-life example involving quotes and escaped quotes (compilation of GCC) args = [ @@ -2657,8 +2657,8 @@ def test_rpath_args_script(self): ] cmd = "%s g++ '' '%s' %s" % (script, rpath_inc, ' '.join(args)) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, @@ -2678,15 +2678,16 @@ def test_rpath_args_script(self): "'-o' 'build/version.o'", "'../../gcc/version.c'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # verify that no -rpath arguments are injected when command is run in 'version check' mode for extra_args in ["-v", "-V", "--version", "-dumpversion", "-v -L/test/lib"]: cmd = "%s g++ '' '%s' %s" % (script, rpath_inc, extra_args) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(["'%s'" % x for x in extra_args.split(' ')])) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) + cmd_args = ' '.join(["'%s'" % x for x in extra_args.split(' ')]) + self.assertEqual(res.output.strip(), f"CMD_ARGS=({cmd_args})") # if a compiler command includes "-x c++-header" or "-x c-header" (which imply no linking is done), # we should *not* inject -Wl,-rpath options, since those enable linking as a side-effect; @@ -2699,10 +2700,10 @@ def test_rpath_args_script(self): for extra_args in test_cases: cmd = "%s g++ '' '%s' foo.c -O2 %s" % (script, rpath_inc, extra_args) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) cmd_args = ["'foo.c'", "'-O2'"] + ["'%s'" % x for x in extra_args.split(' ')] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # check whether $LIBRARY_PATH is taken into account test_cmd_gcc = "%s gcc '' '%s' -c foo.c" % (script, rpath_inc) @@ -2768,16 +2769,16 @@ def test_rpath_args_script(self): os.environ['LIBRARY_PATH'] = ':'.join(library_path) with self.mocked_stdout_stderr(): - out, ec = run_cmd(test_cmd_gcc, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(test_cmd_gcc) + self.assertEqual(res.exit_code, 0) cmd_args = pre_cmd_args_gcc + ["'-Wl,-rpath=%s'" % x for x in library_path if x] + post_cmd_args_gcc - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) with self.mocked_stdout_stderr(): - out, ec = run_cmd(test_cmd_ld, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(test_cmd_ld) + self.assertEqual(res.exit_code, 0) cmd_args = pre_cmd_args_ld + ["'-rpath=%s'" % x for x in library_path if x] + post_cmd_args_ld - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # paths already listed via -L don't get included again as RPATH option new_lib64 = os.path.join(self.test_prefix, 'new', 'lib64') @@ -2796,19 +2797,19 @@ def test_rpath_args_script(self): os.environ['LIBRARY_PATH'] = ':'.join(library_path) with self.mocked_stdout_stderr(): - out, ec = run_cmd(test_cmd_gcc, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(test_cmd_gcc) + self.assertEqual(res.exit_code, 0) # no -L options in GCC command, so all $LIBRARY_PATH entries are retained except for last one (lib symlink) cmd_args = pre_cmd_args_gcc + ["'-Wl,-rpath=%s'" % x for x in library_path[:-1] if x] + post_cmd_args_gcc - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) with self.mocked_stdout_stderr(): - out, ec = run_cmd(test_cmd_ld, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(test_cmd_ld) + self.assertEqual(res.exit_code, 0) # only new path from $LIBRARY_PATH is included as -rpath option, # since others are already included via corresponding -L flag cmd_args = pre_cmd_args_ld + ["'-rpath=%s'" % new_lib64] + post_cmd_args_ld - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) def test_toolchain_prepare_rpath(self): """Test toolchain.prepare under --rpath""" @@ -2926,8 +2927,8 @@ def test_toolchain_prepare_rpath(self): '-DX="\\"\\""', ]) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) expected = ' '.join([ '-Wl,--disable-new-dtags', '-Wl,-rpath=%s/foo' % self.test_prefix, @@ -2937,7 +2938,7 @@ def test_toolchain_prepare_rpath(self): '$FOO', '-DX=""', ]) - self.assertEqual(out.strip(), expected % {'user': os.getenv('USER')}) + self.assertEqual(res.output.strip(), expected % {'user': os.getenv('USER')}) # check whether 'stubs' library directory are correctly filtered out paths = [ @@ -2963,8 +2964,8 @@ def test_toolchain_prepare_rpath(self): cmd = "g++ ${USER}.c %s" % ' '.join(args) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) expected = ' '.join([ '-Wl,--disable-new-dtags', @@ -2989,7 +2990,7 @@ def test_toolchain_prepare_rpath(self): '-L%s/prefix/software/bleh/0/lib/stubs' % self.test_prefix, '-L%s/prefix/software/foobar/4.5/stubsbutnotreally' % self.test_prefix, ]) - self.assertEqual(out.strip(), expected % {'user': os.getenv('USER')}) + self.assertEqual(res.output.strip(), expected % {'user': os.getenv('USER')}) # calling prepare() again should *not* result in wrapping the existing RPATH wrappers # this can happen when building extensions From 47f7a20941becaa44397674cba99b4a7f1dc8617 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 4 Jan 2024 13:25:33 +0100 Subject: [PATCH 258/312] switch to run_shell_cmd where possible in easybuild.* modules --- easybuild/framework/extension.py | 8 ++++---- easybuild/toolchains/linalg/flexiblas.py | 6 +++--- easybuild/tools/job/slurm.py | 18 +++++++++--------- easybuild/tools/options.py | 7 ++++--- easybuild/tools/package/utilities.py | 4 ++-- 5 files changed, 22 insertions(+), 21 deletions(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 8dc4669392..d30242495c 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -42,7 +42,7 @@ from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict from easybuild.tools.build_log import EasyBuildError, raise_nosupport from easybuild.tools.filetools import change_dir -from easybuild.tools.run import check_async_cmd, run_cmd +from easybuild.tools.run import check_async_cmd, run_cmd, run_shell_cmd def resolve_exts_filter_template(exts_filter, ext): @@ -274,14 +274,14 @@ def sanity_check_step(self): elif exts_filter: cmd, stdin = resolve_exts_filter_template(exts_filter, self) # set log_ok to False so we can catch the error instead of run_cmd - (output, ec) = run_cmd(cmd, log_ok=False, simple=False, regexp=False, inp=stdin) + cmd_res = run_shell_cmd(cmd, fail_on_error=False, stdin=stdin) - if ec: + if cmd_res.exit_code: if stdin: fail_msg = 'command "%s" (stdin: "%s") failed' % (cmd, stdin) else: fail_msg = 'command "%s" failed' % cmd - fail_msg += "; output:\n%s" % output.strip() + fail_msg += "; output:\n%s" % cmd_res.output.strip() self.log.warning("Sanity check for '%s' extension failed: %s", self.name, fail_msg) res = (False, fail_msg) # keep track of all reasons of failure diff --git a/easybuild/toolchains/linalg/flexiblas.py b/easybuild/toolchains/linalg/flexiblas.py index c266aff248..3726e7444f 100644 --- a/easybuild/toolchains/linalg/flexiblas.py +++ b/easybuild/toolchains/linalg/flexiblas.py @@ -34,7 +34,7 @@ from easybuild.tools.toolchain.linalg import LinAlg -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext @@ -48,11 +48,11 @@ def det_flexiblas_backend_libs(): # System-wide (config directory): # OPENBLAS # library = libflexiblas_openblas.so - out, _ = run_cmd("flexiblas list", simple=False, trace=False) + res = run_shell_cmd("flexiblas list", hidden=True) shlib_ext = get_shared_lib_ext() flexiblas_lib_regex = re.compile(r'library = (?Plib.*\.%s)' % shlib_ext, re.M) - flexiblas_libs = flexiblas_lib_regex.findall(out) + flexiblas_libs = flexiblas_lib_regex.findall(res.output) backend_libs = [] for flexiblas_lib in flexiblas_libs: diff --git a/easybuild/tools/job/slurm.py b/easybuild/tools/job/slurm.py index 1f2dea776d..97a925bd35 100644 --- a/easybuild/tools/job/slurm.py +++ b/easybuild/tools/job/slurm.py @@ -37,7 +37,7 @@ from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, build_option from easybuild.tools.job.backend import JobBackend from easybuild.tools.filetools import which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd _log = fancylogger.getLogger('slurm', fname=False) @@ -78,8 +78,8 @@ def __init__(self, *args, **kwargs): def _check_version(self): """Check whether version of Slurm complies with required version.""" - (out, _) = run_cmd("sbatch --version", trace=False) - slurm_ver = out.strip().split(' ')[-1] + res = run_shell_cmd("sbatch --version", hidden=True) + slurm_ver = res.output.strip().split(' ')[-1] self.log.info("Found Slurm version %s", slurm_ver) if LooseVersion(slurm_ver) < LooseVersion(self.REQ_VERSION): @@ -116,16 +116,16 @@ def queue(self, job, dependencies=frozenset()): else: submit_cmd += ' --%s "%s"' % (key, job.job_specs[key]) - (out, _) = run_cmd(submit_cmd, trace=False) + cmd_res = run_shell_cmd(submit_cmd, hidden=True) jobid_regex = re.compile("^Submitted batch job (?P[0-9]+)") - res = jobid_regex.search(out) - if res: - job.jobid = res.group('jobid') + regex_res = jobid_regex.search(cmd_res.output) + if regex_res: + job.jobid = regex_res.group('jobid') self.log.info("Job submitted, got job ID %s", job.jobid) else: - raise EasyBuildError("Failed to determine job ID from output of submission command: %s", out) + raise EasyBuildError("Failed to determine job ID from output of submission command: %s", cmd_res.output) self._submitted.append(job) @@ -142,7 +142,7 @@ def complete(self): job_ids.append(job.jobid) if job_ids: - run_cmd("scontrol release %s" % ' '.join(job_ids), trace=False) + run_shell_cmd("scontrol release %s" % ' '.join(job_ids), hidden=True) submitted_jobs = '; '.join(["%s (%s): %s" % (job.name, job.module, job.jobid) for job in self._submitted]) print_msg("List of submitted jobs (%d): %s" % (len(self._submitted), submitted_jobs), log=self.log) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 0d2e5eb853..be5cf94f75 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -98,7 +98,7 @@ from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes from easybuild.tools.modules import Lmod from easybuild.tools.robot import det_robot_path -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.package.utilities import avail_package_naming_schemes from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, OPTARCH_MAP_CHAR, OPTARCH_SEP, Compiler from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME @@ -1893,8 +1893,9 @@ def set_tmpdir(tmpdir=None, raise_error=False): fd, tmptest_file = tempfile.mkstemp() os.close(fd) os.chmod(tmptest_file, 0o700) - if not run_cmd(tmptest_file, simple=True, log_ok=False, regexp=False, force_in_dry_run=True, trace=False, - stream_output=False, with_hooks=False): + res = run_shell_cmd(tmptest_file, fail_on_error=False, in_dry_run=True, hidden=True, stream_output=False, + with_hooks=False) + if res.exit_code: msg = "The temporary directory (%s) does not allow to execute files. " % tempfile.gettempdir() msg += "This can cause problems in the build process, consider using --tmpdir." if raise_error: diff --git a/easybuild/tools/package/utilities.py b/easybuild/tools/package/utilities.py index 9a695e33da..adaa0814fc 100644 --- a/easybuild/tools/package/utilities.py +++ b/easybuild/tools/package/utilities.py @@ -45,7 +45,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import change_dir, which from easybuild.tools.package.package_naming_scheme.pns import PackageNamingScheme -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.utilities import get_subclasses, import_available_modules @@ -145,7 +145,7 @@ def package_with_fpm(easyblock): ]) cmd = ' '.join(cmdlist) _log.debug("The flattened cmdlist looks like: %s", cmd) - run_cmd(cmdlist, log_all=True, simple=True, shell=False) + run_shell_cmd(cmdlist, use_bash=False) _log.info("Created %s package(s) in %s", pkgtype, workdir) From a6cc6efb11cb8db8af151694ddd1e73dd73ff9ce Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 4 Jan 2024 13:26:15 +0100 Subject: [PATCH 259/312] update test_index_functions to only build index for subdirectory with easyconfig files (excl. tempoary log files) --- test/framework/filetools.py | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 5d127914e7..800ee9ba0b 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2421,12 +2421,13 @@ def test_index_functions(self): self.assertTrue(fp.endswith('.eb') or os.path.basename(fp) == 'checksums.json') # set up some files to create actual index file for - ft.copy_dir(os.path.join(test_ecs, 'g'), os.path.join(self.test_prefix, 'g')) + ecs_dir = os.path.join(self.test_prefix, 'easyconfigs') + ft.copy_dir(os.path.join(test_ecs, 'g'), ecs_dir) # test dump_index function - index_fp = ft.dump_index(self.test_prefix) + index_fp = ft.dump_index(ecs_dir) self.assertExists(index_fp) - self.assertTrue(os.path.samefile(self.test_prefix, os.path.dirname(index_fp))) + self.assertTrue(os.path.samefile(ecs_dir, os.path.dirname(index_fp))) datestamp_pattern = r"[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+" expected_header = [ @@ -2434,9 +2435,9 @@ def test_index_functions(self): "# valid until: " + datestamp_pattern, ] expected = [ - os.path.join('g', 'gzip', 'gzip-1.4.eb'), - os.path.join('g', 'GCC', 'GCC-7.3.0-2.30.eb'), - os.path.join('g', 'gompic', 'gompic-2018a.eb'), + os.path.join('gzip', 'gzip-1.4.eb'), + os.path.join('GCC', 'GCC-7.3.0-2.30.eb'), + os.path.join('gompic', 'gompic-2018a.eb'), ] index_txt = ft.read_file(index_fp) for fn in expected_header + expected: @@ -2446,28 +2447,28 @@ def test_index_functions(self): # test load_index function self.mock_stderr(True) self.mock_stdout(True) - index = ft.load_index(self.test_prefix) + index = ft.load_index(ecs_dir) stderr = self.get_stderr() stdout = self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) self.assertFalse(stderr) - regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % self.test_prefix) + regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % ecs_dir) self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout)) - self.assertEqual(len(index), 26) + self.assertEqual(len(index), 25) for fn in expected: self.assertIn(fn, index) # dump_index will not overwrite existing index without force error_pattern = "File exists, not overwriting it without --force" - self.assertErrorRegex(EasyBuildError, error_pattern, ft.dump_index, self.test_prefix) + self.assertErrorRegex(EasyBuildError, error_pattern, ft.dump_index, ecs_dir) ft.remove_file(index_fp) # test creating index file that's infinitely valid - index_fp = ft.dump_index(self.test_prefix, max_age_sec=0) + index_fp = ft.dump_index(ecs_dir, max_age_sec=0) index_txt = ft.read_file(index_fp) expected_header[1] = r"# valid until: 9999-12-31 23:59:59\.9+" for fn in expected_header + expected: @@ -2476,40 +2477,40 @@ def test_index_functions(self): self.mock_stderr(True) self.mock_stdout(True) - index = ft.load_index(self.test_prefix) + index = ft.load_index(ecs_dir) stderr = self.get_stderr() stdout = self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) self.assertFalse(stderr) - regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % self.test_prefix) + regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % ecs_dir) self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout)) - self.assertEqual(len(index), 26) + self.assertEqual(len(index), 25) for fn in expected: self.assertIn(fn, index) ft.remove_file(index_fp) # test creating index file that's only valid for a (very) short amount of time - index_fp = ft.dump_index(self.test_prefix, max_age_sec=1) + index_fp = ft.dump_index(ecs_dir, max_age_sec=1) time.sleep(3) self.mock_stderr(True) self.mock_stdout(True) - index = ft.load_index(self.test_prefix) + index = ft.load_index(ecs_dir) stderr = self.get_stderr() stdout = self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) self.assertIsNone(index) self.assertFalse(stdout) - regex = re.compile(r"WARNING: Index for %s is no longer valid \(too old\), so ignoring it" % self.test_prefix) + regex = re.compile(r"WARNING: Index for %s is no longer valid \(too old\), so ignoring it" % ecs_dir) self.assertTrue(regex.search(stderr), "Pattern '%s' found in: %s" % (regex.pattern, stderr)) # check whether load_index takes into account --ignore-index init_config(build_options={'ignore_index': True}) - self.assertEqual(ft.load_index(self.test_prefix), None) + self.assertEqual(ft.load_index(ecs_dir), None) def test_search_file(self): """Test search_file function.""" From 488d92e571e63256fd1ae94700a28be95d8a5191 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 4 Jan 2024 14:36:02 +0100 Subject: [PATCH 260/312] switch to run_shell_cmd in options test module --- test/framework/options.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 43b5fe66a6..d82b8bf6e9 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -64,7 +64,7 @@ from easybuild.tools.options import EasyBuildOptions, opts_dict_to_eb_opts, parse_external_modules_metadata from easybuild.tools.options import set_up_configuration, set_tmpdir, use_color from easybuild.tools.toolchain.utilities import TC_CONST_PREFIX -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import HAVE_ARCHSPEC from easybuild.tools.version import VERSION from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, cleanup, init_config @@ -3760,8 +3760,8 @@ def test_include_module_naming_schemes(self): # try and make sure top-level directory is in $PYTHONPATH if it isn't yet pythonpath = self.env_pythonpath with self.mocked_stdout_stderr(): - _, ec = run_cmd("cd %s; python -c 'import easybuild.framework'" % self.test_prefix, log_ok=False) - if ec > 0: + res = run_shell_cmd("cd {self.test_prefix}; python -c 'import easybuild.framework'", fail_on_error=False) + if res.exit_code != 0: pythonpath = '%s:%s' % (topdir, pythonpath) fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') @@ -3776,8 +3776,9 @@ def test_include_module_naming_schemes(self): args = ['--avail-module-naming-schemes'] test_cmd = self.mk_eb_test_cmd(args) with self.mocked_stdout_stderr(): - logtxt, _ = run_cmd(test_cmd, simple=False) - self.assertFalse(mns_regex.search(logtxt), "Unexpected pattern '%s' found in: %s" % (mns_regex.pattern, logtxt)) + res = run_shell_cmd(test_cmd) + self.assertFalse(mns_regex.search(res.output), + f"Unexpected pattern '{mns_regex.pattern}' found in: {res.output}") # include extra test MNS mns_txt = '\n'.join([ @@ -3793,8 +3794,9 @@ def test_include_module_naming_schemes(self): args.append('--include-module-naming-schemes=%s/*.py' % self.test_prefix) test_cmd = self.mk_eb_test_cmd(args) with self.mocked_stdout_stderr(): - logtxt, _ = run_cmd(test_cmd, simple=False) - self.assertTrue(mns_regex.search(logtxt), "Pattern '%s' *not* found in: %s" % (mns_regex.pattern, logtxt)) + res = run_shell_cmd(test_cmd) + self.assertTrue(mns_regex.search(res.output), + f"Pattern '{mns_regex.pattern}' *not* found in: {res.output}") def test_use_included_module_naming_scheme(self): """Test using an included module naming scheme.""" @@ -3850,8 +3852,8 @@ def test_include_toolchains(self): # try and make sure top-level directory is in $PYTHONPATH if it isn't yet pythonpath = self.env_pythonpath with self.mocked_stdout_stderr(): - _, ec = run_cmd("cd %s; python -c 'import easybuild.framework'" % self.test_prefix, log_ok=False) - if ec > 0: + res = run_shell_cmd(f"cd {self.test_prefix}; python -c 'import easybuild.framework'", fail_on_error=False) + if res.exit_code != 0: pythonpath = '%s:%s' % (topdir, pythonpath) fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') @@ -3869,8 +3871,9 @@ def test_include_toolchains(self): args = ['--list-toolchains'] test_cmd = self.mk_eb_test_cmd(args) with self.mocked_stdout_stderr(): - logtxt, _ = run_cmd(test_cmd, simple=False) - self.assertFalse(tc_regex.search(logtxt), "Pattern '%s' *not* found in: %s" % (tc_regex.pattern, logtxt)) + res = run_shell_cmd(test_cmd) + self.assertFalse(tc_regex.search(res.output), + f"Pattern '{tc_regex.pattern}' *not* found in: {res.output}") # include extra test toolchain comp_txt = '\n'.join([ @@ -3891,8 +3894,9 @@ def test_include_toolchains(self): args.append('--include-toolchains=%s/*.py,%s/*/*.py' % (self.test_prefix, self.test_prefix)) test_cmd = self.mk_eb_test_cmd(args) with self.mocked_stdout_stderr(): - logtxt, _ = run_cmd(test_cmd, simple=False) - self.assertTrue(tc_regex.search(logtxt), "Pattern '%s' found in: %s" % (tc_regex.pattern, logtxt)) + res = run_shell_cmd(test_cmd) + self.assertTrue(tc_regex.search(res.output), + f"Pattern '{tc_regex.pattern}' found in: {res.output}") def test_cleanup_tmpdir(self): """Test --cleanup-tmpdir.""" @@ -5164,13 +5168,13 @@ def test_dump_env_script(self): self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) with self.mocked_stdout_stderr(): - out, ec = run_cmd("function module { echo $@; } && source %s && echo FC: $FC" % env_script, simple=False) + res = run_shell_cmd(f"function module {{ echo $@; }} && source {env_script} && echo FC: $FC") expected_out = '\n'.join([ "load GCC/4.6.4", "load hwloc/1.11.8-GCC-4.6.4", "FC: gfortran", ]) - self.assertEqual(out.strip(), expected_out) + self.assertEqual(res.output.strip(), expected_out) def test_stop(self): """Test use of --stop.""" From 060b9c3d0e3d00658ee66f432d5616701cd15e65 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 4 Jan 2024 14:41:39 +0100 Subject: [PATCH 261/312] switch to run_shell_cmd in remaining test modules --- test/framework/package.py | 2 +- test/framework/repository.py | 6 +++--- .../sandbox/easybuild/easyblocks/generic/toy_extension.py | 4 ++-- test/framework/sandbox/easybuild/easyblocks/l/libtoy.py | 8 ++++---- test/framework/sandbox/easybuild/easyblocks/t/toy.py | 8 ++++---- test/framework/toy_build.py | 2 +- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/test/framework/package.py b/test/framework/package.py index 38f24242a7..7298d6ecc9 100644 --- a/test/framework/package.py +++ b/test/framework/package.py @@ -44,7 +44,7 @@ FPM_OUTPUT_FILE = 'fpm_mocked.out' -# purposely using non-bash script, to detect issues with shebang line being ignored (run_cmd with shell=False) +# purposely using non-bash script, to detect issues with shebang line being ignored (run_shell_cmd with use_bash=False) MOCKED_FPM = """#!/usr/bin/env python import os, sys diff --git a/test/framework/repository.py b/test/framework/repository.py index a10cfeebf8..acc2749c55 100644 --- a/test/framework/repository.py +++ b/test/framework/repository.py @@ -43,7 +43,7 @@ from easybuild.tools.repository.hgrepo import HgRepository from easybuild.tools.repository.svnrepo import SvnRepository from easybuild.tools.repository.repository import init_repository -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.version import VERSION @@ -95,10 +95,10 @@ def test_gitrepo(self): tmpdir = tempfile.mkdtemp() cmd = "cd %s && git clone --bare %s" % (tmpdir, test_repo_url) with self.mocked_stdout_stderr(): - _, ec = run_cmd(cmd, simple=False, log_all=False, log_ok=False) + res = run_shell_cmd(cmd, fail_on_error=False) # skip remainder of test if creating bare git repo didn't work - if ec == 0: + if res.exit_code == 0: repo = GitRepository(os.path.join(tmpdir, 'testrepository.git')) repo.init() toy_ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index 9c700cf779..de8c4c89c1 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -32,7 +32,7 @@ from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.easyblocks.toy import EB_toy, compose_toy_build_cmd from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class Toy_Extension(ExtensionEasyBlock): @@ -67,7 +67,7 @@ def run(self, *args, **kwargs): EB_toy.build_step(self.master, name=self.name, cfg=self.cfg) if self.cfg['toy_ext_param']: - run_cmd(self.cfg['toy_ext_param']) + run_shell_cmd(self.cfg['toy_ext_param']) return self.module_generator.set_environment('TOY_EXT_%s' % self.name.upper().replace('-', '_'), self.name) diff --git a/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py index 50b573649a..2c0ba5a9e5 100644 --- a/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py +++ b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py @@ -30,7 +30,7 @@ import os from easybuild.framework.easyblock import EasyBlock -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext SHLIB_EXT = get_shared_lib_ext() @@ -40,7 +40,7 @@ class EB_libtoy(EasyBlock): """Support for building/installing libtoy.""" def banned_linked_shared_libs(self): - default = '/thiswillnotbethere,libtoytoytoy.%s,toytoytoy' % SHLIB_EXT + default = f'/thiswillnotbethere,libtoytoytoy.{SHLIB_EXT},toytoytoy' return os.getenv('EB_LIBTOY_BANNED_SHARED_LIBS', default).split(',') def required_linked_shared_libs(self): @@ -53,8 +53,8 @@ def configure_step(self, name=None): def build_step(self, name=None, buildopts=None): """Build libtoy.""" - run_cmd('make') + run_shell_cmd('make') def install_step(self, name=None): """Install libtoy.""" - run_cmd('make install PREFIX="%s"' % self.installdir) + run_shell_cmd(f'make install PREFIX="{self.installdir}"') diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index c4614e3333..a1454435e3 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -38,7 +38,7 @@ from easybuild.tools.environment import setvar from easybuild.tools.filetools import mkdir, write_file from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd def compose_toy_build_cmd(cfg, name, prebuildopts, buildopts): @@ -108,7 +108,7 @@ def configure_step(self, name=None, cfg=None): 'echo "Configured"', cfg['configopts'] ]) - run_cmd(cmd) + run_shell_cmd(cmd) if os.path.exists("%s.source" % name): os.rename('%s.source' % name, '%s.c' % name) @@ -124,8 +124,8 @@ def build_step(self, name=None, cfg=None): cmd = compose_toy_build_cmd(self.cfg, name, cfg['prebuildopts'], cfg['buildopts']) # purposely run build command without checking exit code; # we rely on this in test_toy_build_hooks - (out, ec) = run_cmd(cmd, log_ok=False, log_all=False) - if ec: + res = run_shell_cmd(cmd, fail_on_error=False) + if res.exit_code: print_warning("Command '%s' failed, but we'll ignore it..." % cmd) def install_step(self, name=None): diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index d4c7988534..ca53efc610 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2963,7 +2963,7 @@ def test_toy_build_trace(self): r"^ >> running command:", r"\t\[started at: .*\]", r"\t\[working dir: .*\]", - r"\t\[output logged in .*\]", + r"\t\[output saved to .*\]", r"\tgcc toy.c -o toy\n" r'', ]), From ed8ac7acd85c124caf20484ee2a98e3969a8c333 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 10 Jan 2024 09:44:53 +0100 Subject: [PATCH 262/312] add params modextrapaths_append and allow_append_abs_path --- easybuild/framework/easyblock.py | 8 ++++++++ easybuild/framework/easyconfig/default.py | 2 ++ easybuild/framework/easyconfig/format/format.py | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 9f3233c265..063b2c01cc 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1421,6 +1421,14 @@ def make_module_extra(self, altroot=None, altversion=None): value, type(value)) lines.append(self.module_generator.prepend_paths(key, value, allow_abs=self.cfg['allow_prepend_abs_path'])) + for (key, value) in self.cfg['modextrapaths_append'].items(): + if isinstance(value, string_type): + value = [value] + elif not isinstance(value, (tuple, list)): + raise EasyBuildError("modextrapaths_append dict value %s (type: %s) is not a list or tuple", + value, type(value)) + lines.append(self.module_generator.append_paths(key, value, allow_abs=self.cfg['allow_append_abs_path'])) + modloadmsg = self.cfg['modloadmsg'] if modloadmsg: # add trailing newline to prevent that shell prompt is 'glued' to module load message diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index dd91229d1e..dae27aca65 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -189,10 +189,12 @@ 'exts_list': [[], 'List with extensions added to the base installation', EXTENSIONS], # MODULES easyconfig parameters + 'allow_append_abs_path': [False, "Allow specifying absolute paths to append in modextrapaths_append", MODULES], 'allow_prepend_abs_path': [False, "Allow specifying absolute paths to prepend in modextrapaths", MODULES], 'include_modpath_extensions': [True, "Include $MODULEPATH extensions specified by module naming scheme.", MODULES], 'modaliases': [{}, "Aliases to be defined in module file", MODULES], 'modextrapaths': [{}, "Extra paths to be prepended in module file", MODULES], + 'modextrapaths_append': [{}, "Extra paths to be appended in module file", MODULES], 'modextravars': [{}, "Extra environment variables to be added to module file", MODULES], 'modloadmsg': [{}, "Message that should be printed when generated module is loaded", MODULES], 'modunloadmsg': [{}, "Message that should be printed when generated module is unloaded", MODULES], diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index 9a1626c60b..d503b6703a 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -73,7 +73,7 @@ ] LAST_PARAMS = ['exts_default_options', 'exts_list', 'sanity_check_paths', 'sanity_check_commands', - 'modextrapaths', 'modextravars', + 'modextrapaths', 'modextrapaths_append', 'modextravars', 'moduleclass'] SANITY_CHECK_PATHS_DIRS = 'dirs' From 5b0d6a302501088c700fd6255a1a42bf0b7f40d4 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 11 Jan 2024 13:24:49 +0100 Subject: [PATCH 263/312] Improve --check-github output Fix misdetecting an empty username as valid. Show more information on why/what failed, e.g. for creating gists or not/partially populated --git_working_dirs_path --- easybuild/tools/github.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index ca013743d7..8100c8613b 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -2008,17 +2008,18 @@ def check_github(): github_user = build_option('github_user') github_account = build_option('github_org') or build_option('github_user') - if github_user is None: - check_res = "(none available) => FAIL" - status['--new-pr'] = status['--update-pr'] = status['--upload-test-report'] = False - else: + if github_user: check_res = "%s => OK" % github_user + else: + check_res = "%s => FAIL" % ('(none available)' if github_user is None else '(empty)') + status['--new-pr'] = status['--update-pr'] = status['--upload-test-report'] = False print_msg(check_res, log=_log, prefix=False) # check GitHub token print_msg("* GitHub token...", log=_log, prefix=False, newline=False) github_token = fetch_github_token(github_user) + github_token_valid = False if github_token is None: check_res = "(no token found) => FAIL" else: @@ -2027,6 +2028,7 @@ def check_github(): token_descr = partial_token + " (len: %d)" % len(github_token) if validate_github_token(github_token, github_user): check_res = "%s => OK (validated)" % token_descr + github_token_valid = True else: check_res = "%s => FAIL (validation failed)" % token_descr @@ -2119,7 +2121,7 @@ def check_github(): try: getattr(git_repo.remotes, remote_name).push(branch_name, delete=True) except GitCommandError as err: - sys.stderr.write("WARNING: failed to delete test branch from GitHub: %s\n" % err) + print_warning("failed to delete test branch from GitHub: %s" % err, log=_log) # test creating a gist print_msg("* creating gists...", log=_log, prefix=False, newline=False) @@ -2137,17 +2139,33 @@ def check_github(): if gist_url and re.match('https://gist.github.com/%s/[0-9a-f]+$' % github_user, gist_url): check_res = "OK" - else: + elif not github_user: + check_res = "FAIL (no GitHub user specified)" + elif not github_token: + check_res = "FAIL (missing github token)" + elif not github_token_valid: + check_res = "FAIL (invalid github token)" + elif gist_url: check_res = "FAIL (gist_url: %s)" % gist_url - status['--upload-test-report'] = False + else: + check_res = "FAIL" + if 'FAIL' in check_res: + status['--upload-test-report'] = False print_msg(check_res, log=_log, prefix=False) # check whether location to local working directories for Git repositories is available (not strictly needed) print_msg("* location to Git working dirs... ", log=_log, prefix=False, newline=False) git_working_dirs_path = build_option('git_working_dirs_path') if git_working_dirs_path: - check_res = "OK (%s)" % git_working_dirs_path + repos = [GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO, GITHUB_FRAMEWORK_REPO] + missing_repos = [repo for repo in repos if not os.path.exists(os.path.join(git_working_dirs_path, repo))] + if not missing_repos: + check_res = "OK (%s)" % git_working_dirs_path + elif missing_repos != repos: + check_res = "OK (%s) but missing %s (suboptimal)" % (git_working_dirs_path, ', '.join(missing_repos)) + else: + check_res = "set (%s) but not populated (suboptimal)" % git_working_dirs_path else: check_res = "not found (suboptimal)" From 8127a99c775061f85858fa4b3646cd810df33b00 Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 12 Jan 2024 10:03:47 +0100 Subject: [PATCH 264/312] add tests --- test/framework/easyblock.py | 19 +++++++++++++++++++ test/framework/easyconfig.py | 2 ++ test/framework/toy_build.py | 9 +++++++++ 3 files changed, 30 insertions(+) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 5e2407575d..df40f42d98 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -645,6 +645,7 @@ def test_make_module_extra(self): # also check how absolute paths specified in modexself.contents = '\n'.join([ self.contents += "\nmodextrapaths = {'TEST_PATH_VAR': ['foo', '/test/absolute/path', 'bar']}" + self.contents += "\nmodextrapaths_append = {'TEST_PATH_VAR': ['foo', '/test/absolute/path', 'bar']}" self.writeEC() ec = EasyConfig(self.eb_file) eb = EasyBlock(ec) @@ -657,6 +658,7 @@ def test_make_module_extra(self): # allow use of absolute paths, and verify contents of module self.contents += "\nallow_prepend_abs_path = True" + self.contents += "\nallow_append_abs_path = True" self.writeEC() ec = EasyConfig(self.eb_file) eb = EasyBlock(ec) @@ -675,6 +677,9 @@ def test_make_module_extra(self): r"^prepend[-_]path.*TEST_PATH_VAR.*root.*foo", r"^prepend[-_]path.*TEST_PATH_VAR.*/test/absolute/path", r"^prepend[-_]path.*TEST_PATH_VAR.*root.*bar", + r"^append[-_]path.*TEST_PATH_VAR.*root.*foo", + r"^append[-_]path.*TEST_PATH_VAR.*/test/absolute/path", + r"^append[-_]path.*TEST_PATH_VAR.*root.*bar", ] for pattern in patterns: self.assertTrue(re.search(pattern, txt, re.M), "Pattern '%s' found in: %s" % (pattern, txt)) @@ -1173,6 +1178,7 @@ def test_make_module_step(self): 'PATH': ('xbin', 'pibin'), 'CPATH': 'pi/include', } + modextrapaths_append = modextrapaths.copy() self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "%s"' % name, @@ -1186,6 +1192,7 @@ def test_make_module_step(self): "hiddendependencies = [('test', '1.2.3'), ('OpenMPI', '2.1.2-GCC-6.4.0-2.28')]", "modextravars = %s" % str(modextravars), "modextrapaths = %s" % str(modextrapaths), + "modextrapaths_append = %s" % str(modextrapaths_append), ]) # test if module is generated correctly @@ -1256,6 +1263,18 @@ def test_make_module_step(self): num_prepends = len(regex.findall(txt)) self.assertEqual(num_prepends, 1, "Expected exactly 1 %s command in %s" % (regex.pattern, txt)) + for (key, vals) in modextrapaths_append.items(): + if isinstance(vals, string_type): + vals = [vals] + for val in vals: + if get_module_syntax() == 'Tcl': + regex = re.compile(r'^append-path\s+%s\s+\$root/%s$' % (key, val), re.M) + elif get_module_syntax() == 'Lua': + regex = re.compile(r'^append_path\("%s", pathJoin\(root, "%s"\)\)$' % (key, val), re.M) + else: + self.fail("Unknown module syntax: %s" % get_module_syntax()) + self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) + for (name, ver) in [('GCC', '6.4.0-2.28')]: if get_module_syntax() == 'Tcl': regex = re.compile(r'^\s*module load %s\s*$' % os.path.join(name, ver), re.M) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index dda5c0a05b..905cf6c15b 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1151,6 +1151,7 @@ def test_templating_constants(self): 'R: %%(rver)s, %%(rmajver)s, %%(rminver)s, %%(rshortver)s', ]), 'modextrapaths = {"PI_MOD_NAME": "%%(module_name)s"}', + 'modextrapaths_append = {"PATH_APPEND": "appended_path"}', 'license_file = HOME + "/licenses/PI/license.txt"', "github_account = 'easybuilders'", ]) % inp @@ -1191,6 +1192,7 @@ def test_templating_constants(self): self.assertEqual(ec['modloadmsg'], expected) self.assertEqual(ec['modunloadmsg'], expected) self.assertEqual(ec['modextrapaths'], {'PI_MOD_NAME': 'PI/3.04-Python-2.7.10'}) + self.assertEqual(ec['modextrapaths_append'], {'PATH_APPEND': 'appended_path'}) self.assertEqual(ec['license_file'], os.path.join(os.environ['HOME'], 'licenses', 'PI', 'license.txt')) # test the escaping insanity here (ie all the crap we allow in easyconfigs) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index cc84ce86d1..e5469dd488 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -285,6 +285,7 @@ def test_toy_tweaked(self): ec_extra = '\n'.join([ "versionsuffix = '-tweaked'", "modextrapaths = {'SOMEPATH': ['foo/bar', 'baz', '']}", + "modextrapaths_append = {'SOMEPATH_APPEND': ['qux/fred', 'thud', '']}", "modextravars = {'FOO': 'bar'}", "modloadmsg = '%s'" % modloadmsg, "modtclfooter = 'puts stderr \"oh hai!\"'", # ignored when module syntax is Lua @@ -319,6 +320,9 @@ def test_toy_tweaked(self): self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root/foo/bar$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root/baz$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^append-path\s*SOMEPATH_APPEND\s*\$root/qux/fred$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^append-path\s*SOMEPATH_APPEND\s*\$root/thud$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^append-path\s*SOMEPATH_APPEND\s*\$root$', toy_module_txt, re.M)) mod_load_msg = r'module-info mode load.*\n\s*puts stderr\s*.*%s$' % modloadmsg_regex_tcl self.assertTrue(re.search(mod_load_msg, toy_module_txt, re.M)) self.assertTrue(re.search(r'^puts stderr "oh hai!"$', toy_module_txt, re.M)) @@ -326,6 +330,11 @@ def test_toy_tweaked(self): self.assertTrue(re.search(r'^setenv\("FOO", "bar"\)', toy_module_txt, re.M)) pattern = r'^prepend_path\("SOMEPATH", pathJoin\(root, "foo/bar"\)\)$' self.assertTrue(re.search(pattern, toy_module_txt, re.M)) + pattern = r'^append_path\("SOMEPATH_APPEND", pathJoin\(root, "qux/fred"\)\)$' + self.assertTrue(re.search(pattern, toy_module_txt, re.M)) + pattern = r'^append_path\("SOMEPATH_APPEND", pathJoin\(root, "thud"\)\)$' + self.assertTrue(re.search(pattern, toy_module_txt, re.M)) + self.assertTrue(re.search(r'^append_path\("SOMEPATH_APPEND", root\)$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", pathJoin\(root, "baz"\)\)$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", root\)$', toy_module_txt, re.M)) mod_load_msg = r'^if mode\(\) == "load" then\n\s*io.stderr:write\(%s\)$' % modloadmsg_regex_lua From 4e0a068e79d223390cd5fec443cc9cc7d0b5c91b Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 12 Jan 2024 10:20:57 +0100 Subject: [PATCH 265/312] fix tests --- test/framework/toy_build.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index e5469dd488..0349652b47 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1523,6 +1523,9 @@ def test_toy_module_fulltxt(self): r'prepend_path\("SOMEPATH", pathJoin\(root, "foo/bar"\)\)', r'prepend_path\("SOMEPATH", pathJoin\(root, "baz"\)\)', r'prepend_path\("SOMEPATH", root\)', + r'append_path\("SOMEPATH_APPEND", pathJoin\(root, "qux/fred"\)\)', + r'append_path\("SOMEPATH_APPEND", pathJoin\(root, "thud"\)\)', + r'append_path\("SOMEPATH_APPEND", root\)', r'', r'if mode\(\) == "load" then', ] + modloadmsg_lua + [ @@ -1561,6 +1564,9 @@ def test_toy_module_fulltxt(self): r'prepend-path SOMEPATH \$root/foo/bar', r'prepend-path SOMEPATH \$root/baz', r'prepend-path SOMEPATH \$root', + r'append-path SOMEPATH_APPEND \$root/qux/fred', + r'append-path SOMEPATH_APPEND \$root/thud', + r'append-path SOMEPATH_APPEND \$root', r'', r'if { \[ module-info mode load \] } {', ] + modloadmsg_tcl + [ From 2eef785818183d8fd3d53423d0e723729c810460 Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 12 Jan 2024 13:13:41 +0100 Subject: [PATCH 266/312] try to fix tests --- test/framework/easyblock.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index df40f42d98..27ece36ca5 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -644,8 +644,8 @@ def test_make_module_extra(self): installver = '3.14-gompi-2018a' # also check how absolute paths specified in modexself.contents = '\n'.join([ - self.contents += "\nmodextrapaths = {'TEST_PATH_VAR': ['foo', '/test/absolute/path', 'bar']}" - self.contents += "\nmodextrapaths_append = {'TEST_PATH_VAR': ['foo', '/test/absolute/path', 'bar']}" + self.contents += "\nmodextrapaths = {'TEST_PATH_VAR_APPEND': ['foo', '/test/absolute/path', 'bar']}" + self.contents += "\nmodextrapaths_append = {'TEST_PATH_VAR_APPEND': ['foo', '/test/absolute/path', 'bar']}" self.writeEC() ec = EasyConfig(self.eb_file) eb = EasyBlock(ec) @@ -677,9 +677,9 @@ def test_make_module_extra(self): r"^prepend[-_]path.*TEST_PATH_VAR.*root.*foo", r"^prepend[-_]path.*TEST_PATH_VAR.*/test/absolute/path", r"^prepend[-_]path.*TEST_PATH_VAR.*root.*bar", - r"^append[-_]path.*TEST_PATH_VAR.*root.*foo", - r"^append[-_]path.*TEST_PATH_VAR.*/test/absolute/path", - r"^append[-_]path.*TEST_PATH_VAR.*root.*bar", + r"^append[-_]path.*TEST_PATH_VAR_APPEND.*root.*foo", + r"^append[-_]path.*TEST_PATH_VAR_APPEND.*/test/absolute/path", + r"^append[-_]path.*TEST_PATH_VAR_APPEND.*root.*bar", ] for pattern in patterns: self.assertTrue(re.search(pattern, txt, re.M), "Pattern '%s' found in: %s" % (pattern, txt)) @@ -1178,7 +1178,7 @@ def test_make_module_step(self): 'PATH': ('xbin', 'pibin'), 'CPATH': 'pi/include', } - modextrapaths_append = modextrapaths.copy() + modextrapaths_append = {'APPEND_PATH': 'append_path'} self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "%s"' % name, From 58d67dbed338aa42ca5fa2076cfe5f0a96eccb21 Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 12 Jan 2024 15:56:09 +0100 Subject: [PATCH 267/312] fix --- test/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 27ece36ca5..6c08f947d9 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -644,7 +644,7 @@ def test_make_module_extra(self): installver = '3.14-gompi-2018a' # also check how absolute paths specified in modexself.contents = '\n'.join([ - self.contents += "\nmodextrapaths = {'TEST_PATH_VAR_APPEND': ['foo', '/test/absolute/path', 'bar']}" + self.contents += "\nmodextrapaths = {'TEST_PATH_VAR': ['foo', '/test/absolute/path', 'bar']}" self.contents += "\nmodextrapaths_append = {'TEST_PATH_VAR_APPEND': ['foo', '/test/absolute/path', 'bar']}" self.writeEC() ec = EasyConfig(self.eb_file) From 571b94dec403a7afd8e4463fe6b1cd8cc59de373 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 15 Jan 2024 15:36:32 +0100 Subject: [PATCH 268/312] Move post install commands after lib64 symlinking --- easybuild/framework/easyblock.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 0f1c6d6f3f..c8967e812c 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3019,12 +3019,6 @@ def post_install_step(self): - run post install commands if any were specified """ - self.run_post_install_commands() - self.apply_post_install_patches() - self.print_post_install_messages() - - self.fix_shebang() - lib_dir = os.path.join(self.installdir, 'lib') lib64_dir = os.path.join(self.installdir, 'lib64') @@ -3045,6 +3039,12 @@ def post_install_step(self): # create *relative* 'lib' symlink to 'lib64'; symlink('lib64', lib_dir, use_abspath_source=False) + self.run_post_install_commands() + self.apply_post_install_patches() + self.print_post_install_messages() + + self.fix_shebang() + def sanity_check_step(self, *args, **kwargs): """ Do a sanity check on the installation From b03f26e9ebaa9de6407b1e40459c1d586c3c97d1 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 15 Jan 2024 16:44:10 +0100 Subject: [PATCH 269/312] Add script for updating local git repos with develop branch Usefull when using `--git-working-dirs-path` and/or with checkouts created by `install-EasyBuild-develop.sh`. --- .../scripts/install-EasyBuild-develop.sh | 9 +- easybuild/scripts/update-EasyBuild-develop.sh | 91 +++++++++++++++++++ 2 files changed, 96 insertions(+), 4 deletions(-) create mode 100755 easybuild/scripts/update-EasyBuild-develop.sh diff --git a/easybuild/scripts/install-EasyBuild-develop.sh b/easybuild/scripts/install-EasyBuild-develop.sh index 4181d8c42a..f88ae280a0 100755 --- a/easybuild/scripts/install-EasyBuild-develop.sh +++ b/easybuild/scripts/install-EasyBuild-develop.sh @@ -14,7 +14,7 @@ print_usage() echo echo " github_username: username on GitHub for which the EasyBuild repositories should be cloned" echo - echo " install_dir: directory were all the EasyBuild files will be installed" + echo " install_dir: directory where all the EasyBuild files will be installed" echo } @@ -79,7 +79,7 @@ EOF # Check for 'help' argument -if [ "$1" = "-h" -o "$1" = "--help" ] ; then +if [ "$1" = "-h" ] || [ "$1" = "--help" ] ; then print_usage exit 0 fi @@ -116,13 +116,14 @@ github_clone_branch "easybuild" "develop" EB_DEVEL_MODULE_NAME="EasyBuild-develop" MODULES_INSTALL_DIR=${INSTALL_DIR}/modules EB_DEVEL_MODULE="${MODULES_INSTALL_DIR}/${EB_DEVEL_MODULE_NAME}" -mkdir -p ${MODULES_INSTALL_DIR} +mkdir -p "${MODULES_INSTALL_DIR}" print_devel_module > "${EB_DEVEL_MODULE}" -echo +echo echo "=== Run 'module use ${MODULES_INSTALL_DIR}' and 'module load ${EB_DEVEL_MODULE_NAME}' to use your development version of EasyBuild." echo "=== (you can append ${MODULES_INSTALL_DIR} to your MODULEPATH to make this module always available for loading)" echo echo "=== To update each repository, run 'git pull origin' in each subdirectory of ${INSTALL_DIR}" +echo "=== Or run $(dirname "$0")/update-EasyBuild-develop.sh '${INSTALL_DIR}'" echo exit 0 diff --git a/easybuild/scripts/update-EasyBuild-develop.sh b/easybuild/scripts/update-EasyBuild-develop.sh new file mode 100755 index 0000000000..c80289ad96 --- /dev/null +++ b/easybuild/scripts/update-EasyBuild-develop.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +# Stop in case of error +set -e + +# Print script help +print_usage() +{ + echo "Checkout develop branch of all EasyBuild repositories" + echo "and pull changes from the remote repository." + echo "To be used with the EasyBuild-develop module or a set git-working-dirs-path" + echo "Usage: $0 []" + echo + echo " git_dir: directory where all the EasyBuild repositories are installed." + echo " Automatically detected if not specified." + echo +} + +if [[ "$1" = "-h" ]] || [[ "$1" = "--help" ]]; then + print_usage + exit 0 +fi + +if [[ $# -gt 1 ]] ; then + echo "Error: invalid arguments" + echo + print_usage + exit 1 +fi + +if [[ $# -eq 1 ]]; then + git_dir=$1 +else + # Auto detect git_dir + git_dir="" + if ! which eb &> /dev/null; then + module load EasyBuild-develop || module load EasyBuild || true + if ! which eb &> /dev/null; then + echo 'Found neither the `eb` command nor a working module.' + echo 'Please specify the git_dir!' + exit 1 + fi + fi + if out=$(eb --show-config | grep -F 'git-working-dirs-path'); then + path=$(echo "$out" | awk '{print $NF}') + if [[ -n "$path" ]] && [[ -d "$path" ]]; then + git_dir=$path + echo "Using git_dir from git-working-dirs-path: $git_dir" + fi + fi + if [[ -z "$git_dir" ]]; then + eb_dir=$(dirname "$(which eb)") + if [[ "$(basename "$eb_dir")" == "easybuild-framework" ]] && [[ -d "$eb_dir/.git" ]]; then + git_dir=$(dirname "$eb_dir") + echo "Using git_dir from eb command: $git_dir" + else + echo 'Please specify the git_dir as auto-detection failed!' + exit 1 + fi + fi +fi + +cd "$git_dir" + +for folder in easybuild easybuild-framework easybuild-easyblocks easybuild-easyconfigs; do + echo # A newline + if [[ -d "$folder" ]]; then + echo "========= Checking ${folder} =========" + else + echo "========= Skipping non-existent ${folder} =========" + fi + cd "$folder" + git checkout "develop" + if git remote | grep -qF github_easybuilders; then + git pull "github_easybuilders" + else + git pull + fi + cd .. +done + +index_file="$git_dir/easybuild-easyconfigs/easybuild/easyconfigs/.eb-path-index" +if [[ -f "$index_file" ]]; then + echo -n "Trying to remove index from ${index_file}..." + if rm "$index_file"; then + echo "Done!" + echo "Recreate with 'eb --create-index \"$(dirname "$index_file")\"'" + else + echo "Failed!" + fi +fi From edeb30a194b09131a0d8785dff77a4547ff601ee Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Mon, 15 Jan 2024 16:05:36 +0000 Subject: [PATCH 270/312] Deprecate EnvironmentModulesC and EnvironmentModulesTcl --- easybuild/tools/modules.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index c0d8c25c59..bdf764e9cc 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1185,6 +1185,7 @@ class EnvironmentModulesC(ModulesTool): COMMAND = "modulecmd" REQ_VERSION = '3.2.10' MAX_VERSION = '3.99' + DEPR_VERSION = '3.999' VERSION_REGEXP = r'^\s*(VERSION\s*=\s*)?(?P\d\S*)\s*' def run_module(self, *args, **kwargs): @@ -1246,6 +1247,7 @@ class EnvironmentModulesTcl(EnvironmentModulesC): COMMAND_SHELL = ['tclsh'] VERSION_OPTION = '' REQ_VERSION = None + DEPR_VERSION = '9999' VERSION_REGEXP = r'^Modules\s+Release\s+Tcl\s+(?P\d\S*)\s' def set_path_env_var(self, key, paths): From 289ff54b64b90afce798a9d8bdf34b75251f1009 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Mon, 15 Jan 2024 16:12:18 +0000 Subject: [PATCH 271/312] update deprecation message --- easybuild/tools/modules.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index bdf764e9cc..2df2e165e9 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -272,11 +272,7 @@ def set_and_check_version(self): if StrictVersion(self.version) < StrictVersion(self.DEPR_VERSION): depr_msg = "Support for %s version < %s is deprecated, " % (self.NAME, self.DEPR_VERSION) depr_msg += "found version %s" % self.version - - if self.version.startswith('6') and 'Lmod6' in build_option('silence_deprecation_warnings'): - self.log.warning(depr_msg) - else: - self.log.deprecated(depr_msg, '5.0') + self.log.deprecated(depr_msg, '6.0') if self.MAX_VERSION is not None: self.log.debug("Maximum allowed %s version defined: %s", self.NAME, self.MAX_VERSION) From bd99f6578967e28a8a1cbc99d1f9f5029cd054cd Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 16 Jan 2024 10:14:03 +0100 Subject: [PATCH 272/312] Error when multiple PR options are passed Passing `--preview-pr` in addition to `--new-pr` will not show a preview but actually create a new PR. In general it is always wrong to pass multiple PR-options to EB. So check for that and error out with an appropriate error message. --- easybuild/main.py | 33 +++++++++++++++++++++++---------- test/framework/options.py | 10 ++++++++++ 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 73550e3998..802009166e 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -333,11 +333,23 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session categorized_paths = categorize_files_by_type(eb_args) + pr_options = [ + 'new_branch_github', + 'new_pr', + 'new_pr_from_branch', + 'preview_pr', + 'sync_branch_with_develop', + 'sync_pr_with_develop', + 'update_branch_github', + 'update_pr', + ] + set_pr_options = [opt for opt in pr_options if getattr(options, opt)] + any_pr_option_set = len(set_pr_options) > 0 + if len(set_pr_options) > 1: + raise EasyBuildError("The following options are set but incompatible: %s.\nRemove at least one!", + ', '.join(['--' + opt.replace('_', '-') for opt in set_pr_options])) # command line options that do not require any easyconfigs to be specified - pr_options = options.new_branch_github or options.new_pr or options.new_pr_from_branch or options.preview_pr - pr_options = pr_options or options.sync_branch_with_develop or options.sync_pr_with_develop - pr_options = pr_options or options.update_branch_github or options.update_pr - no_ec_opts = [options.aggregate_regtest, options.regtest, pr_options, search_query] + no_ec_opts = [options.aggregate_regtest, options.regtest, any_pr_option_set, search_query] # determine paths to easyconfigs determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) @@ -427,9 +439,10 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session forced = options.force or options.rebuild dry_run_mode = options.dry_run or options.dry_run_short or options.missing_modules - keep_available_modules = forced or dry_run_mode or options.extended_dry_run or pr_options or options.copy_ec - keep_available_modules = keep_available_modules or options.inject_checksums or options.sanity_check_only - keep_available_modules = keep_available_modules or options.inject_checksums_to_json + keep_available_modules = any(( + forced, dry_run_mode, options.extended_dry_run, any_pr_option_set, options.copy_ec, options.inject_checksums, + options.sanity_check_only, options.inject_checksums_to_json) + ) # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping if not keep_available_modules: @@ -448,12 +461,12 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session if len(easyconfigs) > 0: # resolve dependencies if robot is enabled, except in dry run mode # one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled - if options.robot and (not dry_run_mode or pr_options): + if options.robot and (not dry_run_mode or any_pr_option_set): print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, modtool) else: ordered_ecs = easyconfigs - elif pr_options: + elif any_pr_option_set: ordered_ecs = None else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) @@ -472,7 +485,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session return True # creating/updating PRs - if pr_options: + if any_pr_option_set: if options.new_pr: new_pr(categorized_paths, ordered_ecs) elif options.new_branch_github: diff --git a/test/framework/options.py b/test/framework/options.py index 4b5e1afce8..0067525bde 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3915,6 +3915,16 @@ def test_github_review_pr(self): self.mock_stderr(False) self.assertNotIn("2016.04", txt) + def test_set_multiple_pr_opts(self): + """Test that passing multiple PR options results in an error""" + test_cases = [ + ['--new-pr', 'dummy.eb', '--preview-pr'], + ['--new-pr', 'dummy.eb', '--update-pr', '42'], + ] + for args in test_cases: + error_pattern = "The following options are set but incompatible.* " + args[0] + self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, args, raise_error=True) + def test_set_tmpdir(self): """Test set_tmpdir config function.""" self.purge_environment() From 1da0e55c256d63fb75fce06772b39cff270e54cf Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 16 Jan 2024 10:25:08 +0100 Subject: [PATCH 273/312] Remove pr_options variable Use a tuple in the list generator to avoid accidentally using that variable which now contained ALL options not if any one was set (as before). --- easybuild/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 802009166e..90ee5882fc 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -333,7 +333,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session categorized_paths = categorize_files_by_type(eb_args) - pr_options = [ + set_pr_options = [opt for opt in ( 'new_branch_github', 'new_pr', 'new_pr_from_branch', @@ -342,8 +342,8 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session 'sync_pr_with_develop', 'update_branch_github', 'update_pr', + ) if getattr(options, opt) ] - set_pr_options = [opt for opt in pr_options if getattr(options, opt)] any_pr_option_set = len(set_pr_options) > 0 if len(set_pr_options) > 1: raise EasyBuildError("The following options are set but incompatible: %s.\nRemove at least one!", From a7961568c82c945f4546aa07d88e9841378fb819 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 16 Jan 2024 12:56:33 +0100 Subject: [PATCH 274/312] don't hardcode /bin/bash in eb script, RPATH wrapper script, and run_shell_cmd --- easybuild/scripts/rpath_wrapper_template.sh.in | 2 +- easybuild/tools/run.py | 5 ++++- eb | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/easybuild/scripts/rpath_wrapper_template.sh.in b/easybuild/scripts/rpath_wrapper_template.sh.in index 55c3388c5b..73eb21f0e0 100644 --- a/easybuild/scripts/rpath_wrapper_template.sh.in +++ b/easybuild/scripts/rpath_wrapper_template.sh.in @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash ## # Copyright 2016-2023 Ghent University # diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 0bb0d3c9f4..92c92a1b41 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -41,6 +41,7 @@ import os import re import signal +import shutil import subprocess import sys import tempfile @@ -276,7 +277,9 @@ def to_cmd_str(cmd): # (which could be dash instead of bash, like on Ubuntu, see https://wiki.ubuntu.com/DashAsBinSh) # stick to None (default value) when not running command via a shell if use_bash: - executable, shell = '/bin/bash', True + bash = shutil.which('bash') + _log.info(f"Path to bash that will be used to run shell commands: {bash}") + executable, shell = bash, True else: executable, shell = None, False diff --git a/eb b/eb index 402bb87d0a..f427c5e7e7 100755 --- a/eb +++ b/eb @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash ## # Copyright 2009-2023 Ghent University # From 0b68c30ab63f3ce8ed21adb3eb33105c60b2adf2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 17 Jan 2024 14:06:53 +0100 Subject: [PATCH 275/312] fix broken test_extensions_sanity_check which was hardcoding /bin/bash in expected error message --- test/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 9f656573d1..40ed5f692a 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -2082,7 +2082,7 @@ def test_extensions_sanity_check(self): eb.silent = True error_pattern = r"Sanity check failed: extensions sanity check failed for 1 extensions: toy\n" error_pattern += r"failing sanity check for 'toy' extension: " - error_pattern += r'command "thisshouldfail" failed; output:\n/bin/bash:.* thisshouldfail: command not found' + error_pattern += r'command "thisshouldfail" failed; output:\n.* thisshouldfail: command not found' with self.mocked_stdout_stderr(): self.assertErrorRegex(EasyBuildError, error_pattern, eb.run_all_steps, True) From a468ef9f3eb083da8e97285f720bc5e58f28f061 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 17 Jan 2024 17:32:02 +0100 Subject: [PATCH 276/312] Enhance error message --- easybuild/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 90ee5882fc..85501f57f0 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -346,7 +346,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session ] any_pr_option_set = len(set_pr_options) > 0 if len(set_pr_options) > 1: - raise EasyBuildError("The following options are set but incompatible: %s.\nRemove at least one!", + raise EasyBuildError("The following options are set but incompatible: %s.\nYou can only use one at a time!", ', '.join(['--' + opt.replace('_', '-') for opt in set_pr_options])) # command line options that do not require any easyconfigs to be specified no_ec_opts = [options.aggregate_regtest, options.regtest, any_pr_option_set, search_query] From 17bac3147b220b586b2dadd0b0cb4817f2e01c8e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 17 Jan 2024 17:34:11 +0100 Subject: [PATCH 277/312] Add more test cases --- test/framework/options.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index 0067525bde..f81d34fff8 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3920,6 +3920,8 @@ def test_set_multiple_pr_opts(self): test_cases = [ ['--new-pr', 'dummy.eb', '--preview-pr'], ['--new-pr', 'dummy.eb', '--update-pr', '42'], + ['--new-pr', 'dummy.eb', '--sync-pr-with-develop', '42'], + ['--new-pr', 'dummy.eb', '--new-pr-from-branch', 'mybranch'], ] for args in test_cases: error_pattern = "The following options are set but incompatible.* " + args[0] From 8aaaec2904ca2a260e0d4501f6d8da5b553f4af9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 19:13:25 +0100 Subject: [PATCH 278/312] replace string_type with str in easyblock.py --- easybuild/framework/easyblock.py | 2 +- test/framework/easyblock.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index edf4b01f8a..f6077dd5ce 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1413,7 +1413,7 @@ def make_module_extra(self, altroot=None, altversion=None): lines.append(self.module_generator.prepend_paths(key, value, allow_abs=self.cfg['allow_prepend_abs_path'])) for (key, value) in self.cfg['modextrapaths_append'].items(): - if isinstance(value, string_type): + if isinstance(value, str): value = [value] elif not isinstance(value, (tuple, list)): raise EasyBuildError("modextrapaths_append dict value %s (type: %s) is not a list or tuple", diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 3eed2986b2..64c554ca89 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1273,7 +1273,7 @@ def test_make_module_step(self): self.assertEqual(num_prepends, 1, "Expected exactly 1 %s command in %s" % (regex.pattern, txt)) for (key, vals) in modextrapaths_append.items(): - if isinstance(vals, string_type): + if isinstance(vals, str): vals = [vals] for val in vals: if get_module_syntax() == 'Tcl': From 7cba1dcf64fdb750a95bc0371a72300826014c1c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 21:40:28 +0100 Subject: [PATCH 279/312] implement initial support for running shell commands asynchronously using run_shell_cmd --- easybuild/tools/run.py | 40 +++++++++------ test/framework/run.py | 95 +++++++++++++++++++++++++++++------ test/framework/systemtools.py | 4 +- 3 files changed, 107 insertions(+), 32 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 92c92a1b41..c8d0cabf71 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -45,6 +45,7 @@ import subprocess import sys import tempfile +import threading import time from collections import namedtuple from datetime import datetime @@ -79,7 +80,7 @@ RunShellCmdResult = namedtuple('RunShellCmdResult', ('cmd', 'exit_code', 'output', 'stderr', 'work_dir', - 'out_file', 'err_file')) + 'out_file', 'err_file', 'thread_id')) class RunShellCmdError(BaseException): @@ -199,7 +200,7 @@ def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=N :param use_bash: execute command through bash shell (enabled by default) :param output_file: collect command output in temporary output file :param stream_output: stream command output to stdout (auto-enabled with --logtostdout if None) - :param asynchronous: run command asynchronously + :param asynchronous: indicate that command is being run asynchronously :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined) :param qa_patterns: list of 2-tuples with patterns for questions + corresponding answers :param qa_wait_patterns: list of 2-tuples with patterns for non-questions @@ -223,9 +224,6 @@ def to_cmd_str(cmd): return cmd_str # temporarily raise a NotImplementedError until all options are implemented - if asynchronous: - raise NotImplementedError - if qa_patterns or qa_wait_patterns: raise NotImplementedError @@ -235,6 +233,11 @@ def to_cmd_str(cmd): cmd_str = to_cmd_str(cmd) cmd_name = os.path.basename(cmd_str.split(' ')[0]) + thread_id = None + if asynchronous: + thread_id = threading.get_native_id() + _log.info(f"Initiating running of shell command '{cmd_str}' via thread with ID {thread_id}") + # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely if stream_output is None and build_option('logtostdout'): _log.info(f"Auto-enabling streaming output of '{cmd_str}' command because logging to stdout is enabled") @@ -259,16 +262,16 @@ def to_cmd_str(cmd): if not in_dry_run and build_option('extended_dry_run'): if not hidden or verbose_dry_run: silent = build_option('silent') - msg = f" running command \"{cmd_str}\"\n" + msg = f" running shell command \"{cmd_str}\"\n" msg += f" (in {work_dir})" dry_run_msg(msg, silent=silent) return RunShellCmdResult(cmd=cmd_str, exit_code=0, output='', stderr=None, work_dir=work_dir, - out_file=cmd_out_fp, err_file=cmd_err_fp) + out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id) start_time = datetime.now() if not hidden: - cmd_trace_msg(cmd_str, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp) + _cmd_trace_msg(cmd_str, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id) if stream_output: print_msg(f"(streaming) output for command '{cmd_str}':") @@ -293,7 +296,11 @@ def to_cmd_str(cmd): stderr = subprocess.PIPE if split_stderr else subprocess.STDOUT - _log.info(f"Running command '{cmd_str}' in {work_dir}") + log_msg = f"Running shell command '{cmd_str}' in {work_dir}" + if thread_id: + log_msg += f" (via thread with ID {thread_id})" + _log.info(log_msg) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr, stdin=subprocess.PIPE, cwd=work_dir, env=env, shell=shell, executable=executable) @@ -337,7 +344,7 @@ def to_cmd_str(cmd): raise EasyBuildError(f"Failed to dump command output to temporary file: {err}") res = RunShellCmdResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr, work_dir=work_dir, - out_file=cmd_out_fp, err_file=cmd_err_fp) + out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id) # always log command output cmd_name = cmd_str.split(' ')[0] @@ -370,7 +377,7 @@ def to_cmd_str(cmd): return res -def cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp): +def _cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id): """ Helper function to construct and print trace message for command being run @@ -380,11 +387,18 @@ def cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp): :param stdin: stdin input value for command :param cmd_out_fp: path to output file for command :param cmd_err_fp: path to errors/warnings output file for command + :param thread_id: thread ID (None when not running shell command asynchronously) """ start_time = start_time.strftime('%Y-%m-%d %H:%M:%S') + if thread_id: + run_cmd_msg = f"running shell command (asynchronously, thread ID: {thread_id}):" + else: + run_cmd_msg = "running shell command:" + lines = [ - "running command:", + run_cmd_msg, + f"\t{cmd}", f"\t[started at: {start_time}]", f"\t[working dir: {work_dir}]", ] @@ -395,8 +409,6 @@ def cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp): if cmd_err_fp: lines.append(f"\t[errors/warnings saved to {cmd_err_fp}]") - lines.append('\t' + cmd) - trace_msg('\n'.join(lines)) diff --git a/test/framework/run.py b/test/framework/run.py index db74940aec..9a9b02d767 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -41,6 +41,7 @@ import tempfile import textwrap import time +from concurrent.futures import ThreadPoolExecutor from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner from easybuild.base.fancylogger import setLogLevelDebug @@ -248,7 +249,7 @@ def test_run_shell_cmd_log(self): fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') os.close(fd) - regex_start_cmd = re.compile("Running command 'echo hello' in /") + regex_start_cmd = re.compile("Running shell command 'echo hello' in /") regex_cmd_exit = re.compile(r"Shell command completed successfully \(see output above\): echo hello") # command output is always logged @@ -448,7 +449,7 @@ def test_run_cmd_work_dir(self): def test_run_shell_cmd_work_dir(self): """ - Test running command in specific directory with run_shell_cmd function. + Test running shell command in specific directory with run_shell_cmd function. """ orig_wd = os.getcwd() self.assertFalse(os.path.samefile(orig_wd, self.test_prefix)) @@ -615,11 +616,11 @@ def test_run_shell_cmd_trace(self): """Test run_shell_cmd function in trace mode, and with tracing disabled.""" pattern = [ - r"^ >> running command:", + r"^ >> running shell command:", + r"\techo hello", r"\t\[started at: .*\]", r"\t\[working dir: .*\]", r"\t\[output saved to .*\]", - r"\techo hello", r" >> command completed: exit 0, ran in .*", ] @@ -675,11 +676,11 @@ def test_run_shell_cmd_trace_stdin(self): init_config(build_options={'trace': True}) pattern = [ - r"^ >> running command:", + r"^ >> running shell command:", + r"\techo hello", r"\t\[started at: [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]\]", r"\t\[working dir: .*\]", r"\t\[output saved to .*\]", - r"\techo hello", r" >> command completed: exit 0, ran in .*", ] @@ -707,8 +708,8 @@ def test_run_shell_cmd_trace_stdin(self): self.assertEqual(res.output, 'hello') self.assertEqual(res.exit_code, 0) self.assertEqual(stderr, '') - pattern.insert(3, r"\t\[input: hello\]") - pattern[-2] = "\tcat" + pattern.insert(4, r"\t\[input: hello\]") + pattern[1] = "\tcat" regex = re.compile('\n'.join(pattern)) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) @@ -909,7 +910,8 @@ def test_run_shell_cmd_cache(self): # inject value into cache to check whether executing command again really returns cached value with self.mocked_stdout_stderr(): cached_res = RunShellCmdResult(cmd=cmd, output="123456", exit_code=123, stderr=None, - work_dir='/test_ulimit', out_file='/tmp/foo.out', err_file=None) + work_dir='/test_ulimit', out_file='/tmp/foo.out', err_file=None, + thread_id=None) run_shell_cmd.update_cache({(cmd, None): cached_res}) res = run_shell_cmd(cmd) self.assertEqual(res.cmd, cmd) @@ -928,7 +930,8 @@ def test_run_shell_cmd_cache(self): # inject different output for cat with 'foo' as stdin to check whether cached value is used with self.mocked_stdout_stderr(): cached_res = RunShellCmdResult(cmd=cmd, output="bar", exit_code=123, stderr=None, - work_dir='/test_cat', out_file='/tmp/cat.out', err_file=None) + work_dir='/test_cat', out_file='/tmp/cat.out', err_file=None, + thread_id=None) run_shell_cmd.update_cache({(cmd, 'foo'): cached_res}) res = run_shell_cmd(cmd, stdin='foo') self.assertEqual(res.cmd, cmd) @@ -1006,7 +1009,7 @@ def test_run_shell_cmd_dry_run(self): self.assertEqual(res.output, '') self.assertEqual(res.stderr, None) # check dry run output - expected = """ running command "somecommand foo 123 bar"\n""" + expected = """ running shell command "somecommand foo 123 bar"\n""" self.assertIn(expected, stdout) # check enabling 'hidden' @@ -1029,7 +1032,7 @@ def test_run_shell_cmd_dry_run(self): fail_on_error=False, in_dry_run=True) stdout = self.get_stdout() self.mock_stdout(False) - self.assertNotIn('running command "', stdout) + self.assertNotIn('running shell command "', stdout) self.assertNotEqual(res.exit_code, 0) self.assertEqual(res.output, 'done\n') self.assertEqual(res.stderr, None) @@ -1207,7 +1210,7 @@ def test_run_cmd_async(self): "for i in $(seq 1 50)", "do sleep 0.1", "for j in $(seq 1000)", - "do echo foo", + "do echo foo${i}${j}", "done", "done", "echo done", @@ -1257,8 +1260,68 @@ def test_run_cmd_async(self): res = check_async_cmd(*cmd_info, output=res['output']) self.assertEqual(res['done'], True) self.assertEqual(res['exit_code'], 0) - self.assertTrue(res['output'].startswith('start\n')) - self.assertTrue(res['output'].endswith('\ndone\n')) + self.assertEqual(len(res['output']), 435661) + self.assertTrue(res['output'].startswith('start\nfoo11\nfoo12\n')) + self.assertTrue('\nfoo49999\nfoo491000\nfoo501\n' in res['output']) + self.assertTrue(res['output'].endswith('\nfoo501000\ndone\n')) + + def test_run_shell_cmd_async(self): + """Test asynchronously running of a shell command via run_shell_cmd """ + + thread_pool = ThreadPoolExecutor() + + os.environ['TEST'] = 'test123' + env = os.environ.copy() + + test_cmd = "echo 'sleeping...'; sleep 2; echo $TEST" + task = thread_pool.submit(run_shell_cmd, test_cmd, hidden=True, asynchronous=True, env=env) + + # change value of $TEST to check that command is completed with correct environment + os.environ['TEST'] = 'some_other_value' + + # initial poll should result in None, since it takes a while for the command to complete + self.assertEqual(task.done(), False) + + # wait until command is done + while not task.done(): + time.sleep(1) + res = task.result() + + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, 'sleeping...\ntest123\n') + + # check asynchronous running of failing command + error_test_cmd = "echo 'FAIL!' >&2; exit 123" + task = thread_pool.submit(run_shell_cmd, error_test_cmd, hidden=True, fail_on_error=False, asynchronous=True) + time.sleep(1) + res = task.result() + self.assertEqual(res.exit_code, 123) + self.assertEqual(res.output, "FAIL!\n") + self.assertTrue(res.thread_id) + + # also test with a command that produces a lot of output, + # since that tends to lock up things unless we frequently grab some output... + verbose_test_cmd = ';'.join([ + "echo start", + "for i in $(seq 1 50)", + "do sleep 0.1", + "for j in $(seq 1000)", + "do echo foo${i}${j}", + "done", + "done", + "echo done", + ]) + task = thread_pool.submit(run_shell_cmd, verbose_test_cmd, hidden=True, asynchronous=True) + + while not task.done(): + time.sleep(1) + res = task.result() + + self.assertEqual(res.exit_code, 0) + self.assertEqual(len(res.output), 435661) + self.assertTrue(res.output.startswith('start\nfoo11\nfoo12\n')) + self.assertTrue('\nfoo49999\nfoo491000\nfoo501\n' in res.output) + self.assertTrue(res.output.endswith('\nfoo501000\ndone\n')) def test_check_log_for_errors(self): fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') @@ -1373,7 +1436,7 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): def test_run_shell_cmd_with_hooks(self): """ - Test running command with run_shell_cmd function with pre/post run_shell_cmd hooks in place. + Test running shell command with run_shell_cmd function with pre/post run_shell_cmd hooks in place. """ cwd = os.getcwd() diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 5a8d1033a5..5f4b62f09c 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -341,7 +341,7 @@ def mocked_run_shell_cmd(cmd, **kwargs): } if cmd in known_cmds: return RunShellCmdResult(cmd=cmd, exit_code=0, output=known_cmds[cmd], stderr=None, work_dir=os.getcwd(), - out_file=None, err_file=None) + out_file=None, err_file=None, thread_id=None) else: return run_shell_cmd(cmd, **kwargs) @@ -774,7 +774,7 @@ def test_gcc_version_darwin(self): out = "Apple LLVM version 7.0.0 (clang-700.1.76)" cwd = os.getcwd() mocked_run_res = RunShellCmdResult(cmd="gcc --version", exit_code=0, output=out, stderr=None, work_dir=cwd, - out_file=None, err_file=None) + out_file=None, err_file=None, thread_id=None) st.run_shell_cmd = lambda *args, **kwargs: mocked_run_res self.assertEqual(get_gcc_version(), None) From 1c7c8b4f668dd5d9908bfb5cf92a8d88faef20f6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 22:07:41 +0100 Subject: [PATCH 280/312] use ThreadPoolExecutor to asynchronously run shell commands in EasyBlock.skip_extensions_parallel --- easybuild/framework/easyblock.py | 30 ++++++++++++++++++------------ test/framework/toy_build.py | 6 +++--- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f6077dd5ce..9744397b6b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -52,6 +52,7 @@ import tempfile import time import traceback +from concurrent.futures import ThreadPoolExecutor from datetime import datetime import easybuild.tools.environment as env @@ -87,7 +88,7 @@ from easybuild.tools.hooks import MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP from easybuild.tools.hooks import POSTPROC_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP from easybuild.tools.hooks import SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook -from easybuild.tools.run import RunShellCmdError, check_async_cmd, run_cmd, run_shell_cmd +from easybuild.tools.run import RunShellCmdError, run_shell_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version @@ -1818,7 +1819,9 @@ def skip_extensions_parallel(self, exts_filter): self.log.experimental("Skipping installed extensions in parallel") print_msg("skipping installed extensions (in parallel)", log=self.log) - async_cmd_info_cache = {} + thread_pool = ThreadPoolExecutor(max_workers=self.cfg['parallel']) + + async_shell_cmd_tasks = {} running_checks_ids = [] installed_exts_ids = [] exts_queue = list(enumerate(self.ext_instances[:])) @@ -1831,14 +1834,14 @@ def skip_extensions_parallel(self, exts_filter): # first handle completed checks for idx in running_checks_ids[:]: ext_name = self.ext_instances[idx].name - # don't read any output, just check whether command completed - async_cmd_info = check_async_cmd(*async_cmd_info_cache[idx], output_read_size=0, fail_on_error=False) - if async_cmd_info['done']: - out, ec = async_cmd_info['output'], async_cmd_info['exit_code'] - self.log.info("exts_filter result for %s: exit code %s; output: %s", ext_name, ec, out) + # check whether command completed + task = async_shell_cmd_tasks[idx] + if task.done(): + res = task.result() + self.log.info(f"exts_filter result for {ext_name}: exit code {res.exit_code}; output: {res.output}") running_checks_ids.remove(idx) - if ec == 0: - print_msg("skipping extension %s" % ext_name, log=self.log) + if res.exit_code == 0: + print_msg(f"skipping extension {ext_name}", log=self.log) installed_exts_ids.append(idx) checked_exts_cnt += 1 @@ -1847,11 +1850,12 @@ def skip_extensions_parallel(self, exts_filter): self.update_exts_progress_bar(exts_pbar_label) # start additional checks asynchronously - while exts_queue and len(running_checks_ids) < self.cfg['parallel']: + while exts_queue: idx, ext = exts_queue.pop(0) cmd, stdin = resolve_exts_filter_template(exts_filter, ext) - async_cmd_info_cache[idx] = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, - regexp=False, trace=False, asynchronous=True) + task = thread_pool.submit(run_shell_cmd, cmd, stdin=stdin, hidden=True, + fail_on_error=False, asynchronous=True) + async_shell_cmd_tasks[idx] = task running_checks_ids.append(idx) # compose new list of extensions, skip over the ones that are already installed; @@ -1864,6 +1868,8 @@ def skip_extensions_parallel(self, exts_filter): self.ext_instances = retained_ext_instances + thread_pool.shutdown() + def install_extensions(self, install=True): """ Install extensions. diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 6ba47d3587..936fd09c4c 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1917,7 +1917,7 @@ def test_toy_exts_parallel(self): write_file(test_ec, test_ec_txt) args = ['--parallel-extensions-install', '--experimental', '--force', '--parallel=3'] - stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args, raise_error=True) self.assertEqual(stderr, '') # take into account that each of these lines may appear multiple times, @@ -1936,7 +1936,7 @@ def test_toy_exts_parallel(self): # also test skipping of extensions in parallel args.append('--skip') - stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args, raise_error=True) self.assertEqual(stderr, '') # order in which these patterns occur is not fixed, so check them one by one @@ -1962,7 +1962,7 @@ def test_toy_exts_parallel(self): write_file(toy_ext_eb, toy_ext_eb_txt) args[-1] = '--include-easyblocks=%s' % toy_ext_eb - stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args, raise_error=True) self.assertEqual(stderr, '') # take into account that each of these lines may appear multiple times, # in case no progress was made between checks From e91dbfabc4a4eae9500ef4981f9671b53ab0831f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 22:34:38 +0100 Subject: [PATCH 281/312] use concurrent.futures.wait in EasyBlock.skip_extensions_parallel --- easybuild/framework/easyblock.py | 44 +++++++++++++++----------------- easybuild/tools/run.py | 8 +++--- test/framework/run.py | 4 +-- test/framework/systemtools.py | 4 +-- 4 files changed, 29 insertions(+), 31 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 9744397b6b..4f45dbbaab 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -41,7 +41,6 @@ * Davide Vanzo (Vanderbilt University) * Caspar van Leeuwen (SURF) """ - import copy import glob import inspect @@ -52,7 +51,7 @@ import tempfile import time import traceback -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait from datetime import datetime import easybuild.tools.environment as env @@ -1821,42 +1820,41 @@ def skip_extensions_parallel(self, exts_filter): thread_pool = ThreadPoolExecutor(max_workers=self.cfg['parallel']) - async_shell_cmd_tasks = {} - running_checks_ids = [] + async_shell_cmd_tasks = [] installed_exts_ids = [] exts_queue = list(enumerate(self.ext_instances[:])) checked_exts_cnt = 0 exts_cnt = len(self.ext_instances) + done_tasks = [] # asynchronously run checks to see whether extensions are already installed - while exts_queue or running_checks_ids: + while exts_queue or async_shell_cmd_tasks: # first handle completed checks - for idx in running_checks_ids[:]: + for task in done_tasks: + async_shell_cmd_tasks.remove(task) + res = task.result() + idx = res.task_id ext_name = self.ext_instances[idx].name - # check whether command completed - task = async_shell_cmd_tasks[idx] - if task.done(): - res = task.result() - self.log.info(f"exts_filter result for {ext_name}: exit code {res.exit_code}; output: {res.output}") - running_checks_ids.remove(idx) - if res.exit_code == 0: - print_msg(f"skipping extension {ext_name}", log=self.log) - installed_exts_ids.append(idx) - - checked_exts_cnt += 1 - exts_pbar_label = "skipping installed extensions " - exts_pbar_label += "(%d/%d checked)" % (checked_exts_cnt, exts_cnt) - self.update_exts_progress_bar(exts_pbar_label) + self.log.info(f"exts_filter result for {ext_name}: exit code {res.exit_code}; output: {res.output}") + if res.exit_code == 0: + print_msg(f"skipping extension {ext_name}", log=self.log) + installed_exts_ids.append(idx) + + checked_exts_cnt += 1 + exts_pbar_label = "skipping installed extensions " + exts_pbar_label += "(%d/%d checked)" % (checked_exts_cnt, exts_cnt) + self.update_exts_progress_bar(exts_pbar_label) # start additional checks asynchronously while exts_queue: idx, ext = exts_queue.pop(0) cmd, stdin = resolve_exts_filter_template(exts_filter, ext) task = thread_pool.submit(run_shell_cmd, cmd, stdin=stdin, hidden=True, - fail_on_error=False, asynchronous=True) - async_shell_cmd_tasks[idx] = task - running_checks_ids.append(idx) + fail_on_error=False, asynchronous=True, task_id=idx) + async_shell_cmd_tasks.append(task) + + (done_tasks, _) = wait(async_shell_cmd_tasks, timeout=1, return_when=FIRST_COMPLETED) # compose new list of extensions, skip over the ones that are already installed; # note: original order in extensions list should be preserved! diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index c8d0cabf71..852d13a966 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -80,7 +80,7 @@ RunShellCmdResult = namedtuple('RunShellCmdResult', ('cmd', 'exit_code', 'output', 'stderr', 'work_dir', - 'out_file', 'err_file', 'thread_id')) + 'out_file', 'err_file', 'thread_id', 'task_id')) class RunShellCmdError(BaseException): @@ -184,7 +184,7 @@ def cache_aware_func(cmd, *args, **kwargs): @run_shell_cmd_cache def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, use_bash=True, - output_file=True, stream_output=None, asynchronous=False, with_hooks=True, + output_file=True, stream_output=None, asynchronous=False, task_id=None, with_hooks=True, qa_patterns=None, qa_wait_patterns=None): """ Run specified (interactive) shell command, and capture output + exit code. @@ -267,7 +267,7 @@ def to_cmd_str(cmd): dry_run_msg(msg, silent=silent) return RunShellCmdResult(cmd=cmd_str, exit_code=0, output='', stderr=None, work_dir=work_dir, - out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id) + out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id, task_id=task_id) start_time = datetime.now() if not hidden: @@ -344,7 +344,7 @@ def to_cmd_str(cmd): raise EasyBuildError(f"Failed to dump command output to temporary file: {err}") res = RunShellCmdResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr, work_dir=work_dir, - out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id) + out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id, task_id=task_id) # always log command output cmd_name = cmd_str.split(' ')[0] diff --git a/test/framework/run.py b/test/framework/run.py index 9a9b02d767..c86d8635ed 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -911,7 +911,7 @@ def test_run_shell_cmd_cache(self): with self.mocked_stdout_stderr(): cached_res = RunShellCmdResult(cmd=cmd, output="123456", exit_code=123, stderr=None, work_dir='/test_ulimit', out_file='/tmp/foo.out', err_file=None, - thread_id=None) + thread_id=None, task_id=None) run_shell_cmd.update_cache({(cmd, None): cached_res}) res = run_shell_cmd(cmd) self.assertEqual(res.cmd, cmd) @@ -931,7 +931,7 @@ def test_run_shell_cmd_cache(self): with self.mocked_stdout_stderr(): cached_res = RunShellCmdResult(cmd=cmd, output="bar", exit_code=123, stderr=None, work_dir='/test_cat', out_file='/tmp/cat.out', err_file=None, - thread_id=None) + thread_id=None, task_id=None) run_shell_cmd.update_cache({(cmd, 'foo'): cached_res}) res = run_shell_cmd(cmd, stdin='foo') self.assertEqual(res.cmd, cmd) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 5f4b62f09c..6d8395a5fc 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -341,7 +341,7 @@ def mocked_run_shell_cmd(cmd, **kwargs): } if cmd in known_cmds: return RunShellCmdResult(cmd=cmd, exit_code=0, output=known_cmds[cmd], stderr=None, work_dir=os.getcwd(), - out_file=None, err_file=None, thread_id=None) + out_file=None, err_file=None, thread_id=None, task_id=None) else: return run_shell_cmd(cmd, **kwargs) @@ -774,7 +774,7 @@ def test_gcc_version_darwin(self): out = "Apple LLVM version 7.0.0 (clang-700.1.76)" cwd = os.getcwd() mocked_run_res = RunShellCmdResult(cmd="gcc --version", exit_code=0, output=out, stderr=None, work_dir=cwd, - out_file=None, err_file=None, thread_id=None) + out_file=None, err_file=None, thread_id=None, task_id=None) st.run_shell_cmd = lambda *args, **kwargs: mocked_run_res self.assertEqual(get_gcc_version(), None) From 77e077e372c300d698f2f7da7cf005ea7b555977 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 20:00:19 +0100 Subject: [PATCH 282/312] fix error reporting when test step fails --- easybuild/framework/easyblock.py | 5 ++++- test/framework/toy_build.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f6077dd5ce..a684b74347 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2736,7 +2736,10 @@ def _test_step(self): try: self.test_step() except RunShellCmdError as err: - self.report_test_failure(err) + err.print() + ec_path = os.path.basename(self.cfg.path) + error_msg = f"shell command '{err.cmd_name} ...' failed in test step for {ec_path}" + self.report_test_failure(error_msg) def stage_install_step(self): """ diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 6ba47d3587..82bc052a93 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -4124,6 +4124,22 @@ def test_toy_build_info_msg(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def test_toy_failing_test_step(self): + """ + Test behaviour when test step fails, using toy easyconfig. + """ + test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\nruntest = "false"' + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ec_txt) + + error_pattern = r"shell command 'false \.\.\.' failed in test step" + self.assertErrorRegex(EasyBuildError, error_pattern, self.run_test_toy_build_with_output, + ec_file=test_ec, raise_error=True) + def test_eb_crash(self): """ Test behaviour when EasyBuild crashes, for example due to a buggy hook From ea562b8ead14500f1112bf4b4b8cbbbf20d3dcd2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 22:45:17 +0100 Subject: [PATCH 283/312] fix broken tests --- test/framework/filetools.py | 46 ++++++++++++++++++------------------- test/framework/run.py | 2 +- test/framework/toy_build.py | 4 ++-- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 800ee9ba0b..8b72aefc4c 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2290,7 +2290,7 @@ def test_extract_file(self): self.assertTrue(os.path.samefile(path, self.test_prefix)) self.assertNotExists(os.path.join(self.test_prefix, 'toy-0.0')) - self.assertTrue(re.search('running command "tar xzf .*/toy-0.0.tar.gz"', txt)) + self.assertTrue(re.search('running shell command "tar xzf .*/toy-0.0.tar.gz"', txt)) with self.mocked_stdout_stderr(): path = ft.extract_file(toy_tarball, self.test_prefix, forced=True, change_into_dir=False) @@ -2314,7 +2314,7 @@ def test_extract_file(self): self.assertTrue(os.path.samefile(path, self.test_prefix)) self.assertTrue(os.path.samefile(os.getcwd(), self.test_prefix)) self.assertFalse(stderr) - self.assertTrue("running command" in stdout) + self.assertTrue("running shell command" in stdout) # check whether disabling trace output works with self.mocked_stdout_stderr(): @@ -2800,18 +2800,18 @@ def run_check(): } git_repo = {'git_repo': 'git@github.com:easybuilders/testrepository.git'} # Just to make the below shorter expected = '\n'.join([ - r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"', + r' running shell command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"', r" \(in /.*\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in /.*\)", ]) % git_repo run_check() git_config['clone_into'] = 'test123' expected = '\n'.join([ - r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"', + r' running shell command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"', r" \(in /.*\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git test123"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git test123"', r" \(in /.*\)", ]) % git_repo run_check() @@ -2819,19 +2819,19 @@ def run_check(): git_config['recursive'] = True expected = '\n'.join([ - r' running command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"', + r' running shell command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"', r" \(in /.*\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in /.*\)", ]) % git_repo run_check() git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite'] expected = '\n'.join([ - ' running command "git clone --depth 1 --branch tag_for_tests --recursive' + ' running shell command "git clone --depth 1 --branch tag_for_tests --recursive' + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", ]) % git_repo run_check() @@ -2841,11 +2841,11 @@ def run_check(): 'submodule."sha1".active=false', ] expected = '\n'.join([ - ' running command "git -c submodule."fastahack".active=false -c submodule."sha1".active=false' + ' running shell command "git -c submodule."fastahack".active=false -c submodule."sha1".active=false' + ' clone --depth 1 --branch tag_for_tests --recursive' + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", ]) % git_repo run_check() @@ -2854,9 +2854,9 @@ def run_check(): git_config['keep_git_dir'] = True expected = '\n'.join([ - r' running command "git clone --branch tag_for_tests --recursive %(git_repo)s"', + r' running shell command "git clone --branch tag_for_tests --recursive %(git_repo)s"', r" \(in /.*\)", - r' running command "tar cfvz .*/target/test.tar.gz testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz testrepository"', r" \(in /.*\)", ]) % git_repo run_check() @@ -2865,23 +2865,23 @@ def run_check(): del git_config['tag'] git_config['commit'] = '8456f86' expected = '\n'.join([ - r' running command "git clone --no-checkout %(git_repo)s"', + r' running shell command "git clone --no-checkout %(git_repo)s"', r" \(in /.*\)", - r' running command "git checkout 8456f86 && git submodule update --init --recursive"', + r' running shell command "git checkout 8456f86 && git submodule update --init --recursive"', r" \(in /.*/testrepository\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in /.*\)", ]) % git_repo run_check() git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite'] expected = '\n'.join([ - r' running command "git clone --no-checkout %(git_repo)s"', + r' running shell command "git clone --no-checkout %(git_repo)s"', r" \(in .*/tmp.*\)", - ' running command "git checkout 8456f86 && git submodule update --init --recursive' + ' running shell command "git checkout 8456f86 && git submodule update --init --recursive' + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\'"', r" \(in /.*/testrepository\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", ]) % git_repo run_check() @@ -2889,11 +2889,11 @@ def run_check(): del git_config['recursive'] del git_config['recurse_submodules'] expected = '\n'.join([ - r' running command "git clone --no-checkout %(git_repo)s"', + r' running shell command "git clone --no-checkout %(git_repo)s"', r" \(in /.*\)", - r' running command "git checkout 8456f86"', + r' running shell command "git checkout 8456f86"', r" \(in /.*/testrepository\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in /.*\)", ]) % git_repo run_check() diff --git a/test/framework/run.py b/test/framework/run.py index c86d8635ed..1c9b0b2562 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1273,7 +1273,7 @@ def test_run_shell_cmd_async(self): os.environ['TEST'] = 'test123' env = os.environ.copy() - test_cmd = "echo 'sleeping...'; sleep 2; echo $TEST" + test_cmd = "echo 'sleeping...'; sleep 3; echo $TEST" task = thread_pool.submit(run_shell_cmd, test_cmd, hidden=True, asynchronous=True, env=env) # change value of $TEST to check that command is completed with correct environment diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 936fd09c4c..6ab83f67fc 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2975,11 +2975,11 @@ def test_toy_build_trace(self): r"^== fetching files\.\.\.\n >> sources:\n >> .*/toy-0\.0\.tar\.gz \[SHA256: 44332000.*\]$", r"^ >> applying patch toy-0\.0_fix-silly-typo-in-printf-statement\.patch$", r'\n'.join([ - r"^ >> running command:", + r"^ >> running shell command:", + r"\tgcc toy.c -o toy\n" r"\t\[started at: .*\]", r"\t\[working dir: .*\]", r"\t\[output saved to .*\]", - r"\tgcc toy.c -o toy\n" r'', ]), r" >> command completed: exit 0, ran in .*", From a19776a5e5896fc3c98f594a6aca410ed941f652 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 23:04:01 +0100 Subject: [PATCH 284/312] use threading.get_ident as fallback for threading.get_native_id for Python < 3.8 --- easybuild/tools/run.py | 10 ++++++++-- test/framework/run.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 852d13a966..986c541538 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -45,11 +45,17 @@ import subprocess import sys import tempfile -import threading import time from collections import namedtuple from datetime import datetime +try: + # get_native_id is only available in Python >= 3.8 + from threading import get_native_id as get_thread_id +except ImportError: + # get_ident is available in Python >= 3.3 + from threading import get_ident as get_thread_id + import easybuild.tools.asyncprocess as asyncprocess from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, time_str_since @@ -235,7 +241,7 @@ def to_cmd_str(cmd): thread_id = None if asynchronous: - thread_id = threading.get_native_id() + thread_id = get_thread_id() _log.info(f"Initiating running of shell command '{cmd_str}' via thread with ID {thread_id}") # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely diff --git a/test/framework/run.py b/test/framework/run.py index 1c9b0b2562..c86d8635ed 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1273,7 +1273,7 @@ def test_run_shell_cmd_async(self): os.environ['TEST'] = 'test123' env = os.environ.copy() - test_cmd = "echo 'sleeping...'; sleep 3; echo $TEST" + test_cmd = "echo 'sleeping...'; sleep 2; echo $TEST" task = thread_pool.submit(run_shell_cmd, test_cmd, hidden=True, asynchronous=True, env=env) # change value of $TEST to check that command is completed with correct environment From 75fa3ee8e209beee3e291ad77ddfe0b5101e8475 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Mon, 22 Jan 2024 15:42:28 +0000 Subject: [PATCH 285/312] do not test deprecated module tools --- .github/workflows/unit_tests.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 225b3f85c6..324b64630b 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -14,8 +14,6 @@ jobs: runs-on: ubuntu-20.04 outputs: lmod8: Lmod-8.7.6 - modulesTcl: modules-tcl-1.147 - modules3: modules-3.2.10 modules4: modules-4.1.4 steps: - run: "true" @@ -29,8 +27,6 @@ jobs: # use variables defined by 'setup' job above, see also # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context - ${{needs.setup.outputs.lmod8}} - - ${{needs.setup.outputs.modulesTcl}} - - ${{needs.setup.outputs.modules3}} - ${{needs.setup.outputs.modules4}} lc_all: [""] include: @@ -156,11 +152,7 @@ jobs: export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH eb --version # tell EasyBuild which modules tool is available - if [[ ${{matrix.modules_tool}} =~ ^modules-tcl- ]]; then - export EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl - elif [[ ${{matrix.modules_tool}} =~ ^modules-3 ]]; then - export EASYBUILD_MODULES_TOOL=EnvironmentModulesC - elif [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then + if [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then export EASYBUILD_MODULES_TOOL=EnvironmentModules else export EASYBUILD_MODULES_TOOL=Lmod From fffba9694a1440e560ed36c3201f118405c59a3e Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:05:40 +0000 Subject: [PATCH 286/312] test with non-deprecated module tool --- test/framework/modules.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index f8226e8d33..b28564885c 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1531,10 +1531,10 @@ def test_modulecmd_strip_source(self): '#!/bin/bash', # if last argument (${!#})) is --version, print version 'if [ x"${!#}" == "x--version" ]; then', - ' echo 3.2.10', + ' echo 4.2.10', # otherwise, echo Python commands: set $TEST123 and include a faulty 'source' command 'else', - ' echo "source /opt/cray/pe/modules/3.2.10.6/init/bash"', + ' echo "source /opt/cray/pe/modules/4.2.10.6/init/bash"', " echo \"os.environ['TEST123'] = 'test123'\"", 'fi', ]) @@ -1543,7 +1543,7 @@ def test_modulecmd_strip_source(self): os.environ['PATH'] = '%s:%s' % (self.test_prefix, os.getenv('PATH')) - modtool = EnvironmentModulesC() + modtool = EnvironmentModules() modtool.run_module('load', 'test123') self.assertEqual(os.getenv('TEST123'), 'test123') From 7fffcba6a279a8ea3803f162154810a334236d1f Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:08:45 +0000 Subject: [PATCH 287/312] version with a . --- easybuild/tools/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 09524d5ee1..aa8cbf9381 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1243,7 +1243,7 @@ class EnvironmentModulesTcl(EnvironmentModulesC): COMMAND_SHELL = ['tclsh'] VERSION_OPTION = '' REQ_VERSION = None - DEPR_VERSION = '9999' + DEPR_VERSION = '9999.9' VERSION_REGEXP = r'^Modules\s+Release\s+Tcl\s+(?P\d\S*)\s' def set_path_env_var(self, key, paths): From fd18449daae861c5c8b522942d2346dd98fb674f Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Tue, 23 Jan 2024 09:00:18 +0000 Subject: [PATCH 288/312] set a DEPR_VERSION for EnvironmentModules --- easybuild/tools/modules.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index aa8cbf9381..75a1022b50 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1318,6 +1318,7 @@ class EnvironmentModules(EnvironmentModulesTcl): COMMAND = os.path.join(os.getenv('MODULESHOME', 'MODULESHOME_NOT_DEFINED'), 'libexec', 'modulecmd.tcl') COMMAND_ENVIRONMENT = 'MODULES_CMD' REQ_VERSION = '4.0.0' + DEPR_VERSION = '4.0.0' # needs to be set as EnvironmentModules inherits from EnvironmentModulesTcl MAX_VERSION = None VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d[^+\s]*)(\+\S*)?\s' From 8d8029d02fe241e495d2a96311e2c169fb8f0a9d Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Tue, 23 Jan 2024 09:12:57 +0000 Subject: [PATCH 289/312] uses old module tool, so allow deprecated --- test/framework/modules.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index b28564885c..f1a242f54a 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1531,10 +1531,10 @@ def test_modulecmd_strip_source(self): '#!/bin/bash', # if last argument (${!#})) is --version, print version 'if [ x"${!#}" == "x--version" ]; then', - ' echo 4.2.10', + ' echo 3.2.10', # otherwise, echo Python commands: set $TEST123 and include a faulty 'source' command 'else', - ' echo "source /opt/cray/pe/modules/4.2.10.6/init/bash"', + ' echo "source /opt/cray/pe/modules/3.2.10.6/init/bash"', " echo \"os.environ['TEST123'] = 'test123'\"", 'fi', ]) @@ -1543,7 +1543,8 @@ def test_modulecmd_strip_source(self): os.environ['PATH'] = '%s:%s' % (self.test_prefix, os.getenv('PATH')) - modtool = EnvironmentModules() + self.allow_deprecated_behaviour() + modtool = EnvironmentModulesC() modtool.run_module('load', 'test123') self.assertEqual(os.getenv('TEST123'), 'test123') From cc2a47817e41c6fd62b03ec40976c1a6f6f9897b Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Tue, 23 Jan 2024 09:54:57 +0000 Subject: [PATCH 290/312] and capture the dep. warning --- test/framework/modules.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index f1a242f54a..a849148bdf 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1544,8 +1544,9 @@ def test_modulecmd_strip_source(self): os.environ['PATH'] = '%s:%s' % (self.test_prefix, os.getenv('PATH')) self.allow_deprecated_behaviour() - modtool = EnvironmentModulesC() - modtool.run_module('load', 'test123') + with self.mocked_stdout_stderr(): + modtool = EnvironmentModulesC() + modtool.run_module('load', 'test123') self.assertEqual(os.getenv('TEST123'), 'test123') def test_get_setenv_value_from_modulefile(self): From 7315b9f7a6e47515bb255bf2aa0da8cb22595ff2 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 24 Jan 2024 14:25:47 +0100 Subject: [PATCH 291/312] improve findPythonDeps to recognize non-canonical package names For e.g. `ruamel.yaml` the canonical name is `ruamel-yaml` but the package name as recorded is still `ruamel.yaml`. So if the search using the canonical name didn't find anything try again with the original name before failing. Tested on `maggma==0.60.2` --- easybuild/scripts/findPythonDeps.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index d6e496a048..c3607e47c7 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -96,10 +96,13 @@ def get_dep_tree(package_spec, verbose): def find_deps(pkgs, dep_tree): """Recursively resolve dependencies of the given package(s) and return them""" res = [] - for pkg in pkgs: - pkg = canonicalize_name(pkg) + for orig_pkg in pkgs: + pkg = canonicalize_name(orig_pkg) matching_entries = [entry for entry in dep_tree if pkg in (entry['package']['package_name'], entry['package']['key'])] + if not matching_entries: + matching_entries = [entry for entry in dep_tree + if orig_pkg in (entry['package']['package_name'], entry['package']['key'])] if not matching_entries: raise RuntimeError("Found no installed package for '%s' in %s" % (pkg, dep_tree)) if len(matching_entries) > 1: From 36febd8739ab37900577db8d0cdd13ea0e1f48fa Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 29 Jan 2024 15:18:56 +0100 Subject: [PATCH 292/312] use run_shell_cmd to install extensions in parallel --- easybuild/framework/easyblock.py | 33 ++++++++----- easybuild/framework/extension.py | 48 +------------------ easybuild/tools/run.py | 1 + .../easyblocks/generic/toy_extension.py | 11 +++-- .../sandbox/easybuild/easyblocks/t/toy.py | 6 ++- 5 files changed, 36 insertions(+), 63 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 4f45dbbaab..e2d84c162e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -87,7 +87,7 @@ from easybuild.tools.hooks import MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP from easybuild.tools.hooks import POSTPROC_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP from easybuild.tools.hooks import SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook -from easybuild.tools.run import RunShellCmdError, run_shell_cmd +from easybuild.tools.run import RunShellCmdError, raise_run_shell_cmd_error, run_shell_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version @@ -1961,6 +1961,8 @@ def install_extensions_parallel(self, install=True): """ self.log.info("Installing extensions in parallel...") + thread_pool = ThreadPoolExecutor(max_workers=self.cfg['parallel']) + running_exts = [] installed_ext_names = [] @@ -1997,16 +1999,23 @@ def update_exts_progress_bar_helper(running_exts, progress_size): # check for extension installations that have completed if running_exts: - self.log.info("Checking for completed extension installations (%d running)...", len(running_exts)) + self.log.info(f"Checking for completed extension installations ({len(running_exts)} running)...") for ext in running_exts[:]: - if self.dry_run or ext.async_cmd_check(): - self.log.info("Installation of %s completed!", ext.name) - ext.postrun() - running_exts.remove(ext) - installed_ext_names.append(ext.name) - update_exts_progress_bar_helper(running_exts, 1) + if self.dry_run or ext.async_cmd_task.done(): + res = ext.async_cmd_task.result() + if res.exit_code == 0: + self.log.info(f"Installation of extension {ext.name} completed!") + # run post-install method for extension from same working dir as installation of extension + cwd = change_dir(res.work_dir) + ext.postrun() + change_dir(cwd) + running_exts.remove(ext) + installed_ext_names.append(ext.name) + update_exts_progress_bar_helper(running_exts, 1) + else: + raise_run_shell_cmd_error(res) else: - self.log.debug("Installation of %s is still running...", ext.name) + self.log.debug(f"Installation of extension {ext.name} is still running...") # try to start as many extension installations as we can, taking into account number of available cores, # but only consider first 100 extensions still in the queue @@ -2073,9 +2082,9 @@ def update_exts_progress_bar_helper(running_exts, progress_size): rpath_filter_dirs=self.rpath_filter_dirs) if install: ext.prerun() - ext.run_async() + ext.async_cmd_task = ext.run_async(thread_pool) running_exts.append(ext) - self.log.info("Started installation of extension %s in the background...", ext.name) + self.log.info(f"Started installation of extension {ext.name} in the background...") update_exts_progress_bar_helper(running_exts, 0) # print progress info after every iteration (unless that info is already shown via progress bar) @@ -2088,6 +2097,8 @@ def update_exts_progress_bar_helper(running_exts, progress_size): running_ext_names = ', '.join(x.name for x in running_exts[:3]) + ", ..." print_msg(msg % (installed_cnt, exts_cnt, queued_cnt, running_cnt, running_ext_names), log=self.log) + thread_pool.shutdown() + # # MISCELLANEOUS UTILITY FUNCTIONS # diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index d30242495c..9f099eb74c 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -42,7 +42,7 @@ from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict from easybuild.tools.build_log import EasyBuildError, raise_nosupport from easybuild.tools.filetools import change_dir -from easybuild.tools.run import check_async_cmd, run_cmd, run_shell_cmd +from easybuild.tools.run import run_shell_cmd def resolve_exts_filter_template(exts_filter, ext): @@ -150,12 +150,7 @@ def __init__(self, mself, ext, extra_params=None): self.sanity_check_module_loaded = False self.fake_mod_data = None - self.async_cmd_info = None - self.async_cmd_output = None - self.async_cmd_check_cnt = None - # initial read size should be relatively small, - # to avoid hanging for a long time until desired output is available in async_cmd_check - self.async_cmd_read_size = 1024 + self.async_cmd_task = None @property def name(self): @@ -195,44 +190,6 @@ def postrun(self): """ self.master.run_post_install_commands(commands=self.cfg.get('postinstallcmds', [])) - def async_cmd_start(self, cmd, inp=None): - """ - Start installation asynchronously using specified command. - """ - self.async_cmd_output = '' - self.async_cmd_check_cnt = 0 - self.async_cmd_info = run_cmd(cmd, log_all=True, simple=False, inp=inp, regexp=False, asynchronous=True) - - def async_cmd_check(self): - """ - Check progress of installation command that was started asynchronously. - - :return: True if command completed, False otherwise - """ - if self.async_cmd_info is None: - raise EasyBuildError("No installation command running asynchronously for %s", self.name) - elif self.async_cmd_info is False: - self.log.info("No asynchronous command was started for extension %s", self.name) - return True - else: - self.log.debug("Checking on installation of extension %s...", self.name) - # use small read size, to avoid waiting for a long time until sufficient output is produced - res = check_async_cmd(*self.async_cmd_info, output_read_size=self.async_cmd_read_size) - self.async_cmd_output += res['output'] - if res['done']: - self.log.info("Installation of extension %s completed!", self.name) - self.async_cmd_info = None - else: - self.async_cmd_check_cnt += 1 - self.log.debug("Installation of extension %s still running (checked %d times)", - self.name, self.async_cmd_check_cnt) - # increase read size after sufficient checks, - # to avoid that installation hangs due to output buffer filling up... - if self.async_cmd_check_cnt % 10 == 0 and self.async_cmd_read_size < (1024 ** 2): - self.async_cmd_read_size *= 2 - - return res['done'] - @property def required_deps(self): """Return list of required dependencies for this extension.""" @@ -273,7 +230,6 @@ def sanity_check_step(self): self.log.info("modulename set to False for '%s' extension, so skipping sanity check", self.name) elif exts_filter: cmd, stdin = resolve_exts_filter_template(exts_filter, self) - # set log_ok to False so we can catch the error instead of run_cmd cmd_res = run_shell_cmd(cmd, fail_on_error=False, stdin=stdin) if cmd_res.exit_code: diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 986c541538..95391f3821 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -207,6 +207,7 @@ def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=N :param output_file: collect command output in temporary output file :param stream_output: stream command output to stdout (auto-enabled with --logtostdout if None) :param asynchronous: indicate that command is being run asynchronously + :param task_id: task ID for specified shell command (included in return value) :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined) :param qa_patterns: list of 2-tuples with patterns for questions + corresponding answers :param qa_wait_patterns: list of 2-tuples with patterns for non-questions diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index de8c4c89c1..15fe5773aa 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -27,6 +27,7 @@ @author: Kenneth Hoste (Ghent University) """ +import os from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extensioneasyblock import ExtensionEasyBlock @@ -81,22 +82,24 @@ def prerun(self): super(Toy_Extension, self).run(unpack_src=True) EB_toy.configure_step(self.master, name=self.name, cfg=self.cfg) - def run_async(self): + def run_async(self, thread_pool): """ Install toy extension asynchronously. """ + task_id = f'ext_{self.name}_{self.version}' if self.src: cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts']) - self.async_cmd_start(cmd) else: - self.async_cmd_info = False + cmd = f"echo 'no sources for {self.name}'" + + return thread_pool.submit(run_shell_cmd, cmd, asynchronous=True, env=os.environ.copy(), + fail_on_error=False, task_id=task_id) def postrun(self): """ Wrap up installation of toy extension. """ super(Toy_Extension, self).postrun() - EB_toy.install_step(self.master, name=self.name) def sanity_check_step(self, *args, **kwargs): diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index a1454435e3..fb842bfe06 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -163,12 +163,14 @@ def run(self): """ self.build_step() - def run_async(self): + def run_async(self, thread_pool): """ Asynchronous installation of toy as extension. """ cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts']) - self.async_cmd_start(cmd) + task_id = f'ext_{self.name}_{self.version}' + return thread_pool.submit(run_shell_cmd, cmd, asynchronous=True, env=os.environ.copy(), + fail_on_error=False, task_id=task_id) def postrun(self): """ From cedd5ab74a422b2cbaa026d47204a55ee4be540b Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 3 Feb 2024 19:47:46 +0000 Subject: [PATCH 293/312] enable rpath by default --- easybuild/tools/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index be5cf94f75..cb6ec2022f 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -489,7 +489,7 @@ def override_options(self): 'required-linked-shared-libs': ("Comma-separated list of shared libraries (names, file names, or paths) " "which must be linked in all installed binaries/libraries", 'strlist', 'extend', None), - 'rpath': ("Enable use of RPATH for linking with libraries", None, 'store_true', False), + 'rpath': ("Enable use of RPATH for linking with libraries", None, 'store_true', True), 'rpath-filter': ("List of regex patterns to use for filtering out RPATH paths", 'strlist', 'store', None), 'rpath-override-dirs': ("Path(s) to be prepended when linking with RPATH (string, colon-separated)", None, 'store', None), From 54b36337a187bf1f6605216d82d2ee455fee5dbc Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Sun, 4 Feb 2024 13:44:39 +0000 Subject: [PATCH 294/312] disable rpath in two tests --- test/framework/options.py | 1 + test/framework/toolchain.py | 1 + 2 files changed, 2 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index ca8a582365..5d87695661 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4121,6 +4121,7 @@ def test_extended_dry_run(self): '--buildpath=%s' % self.test_buildpath, '--installpath=%s' % self.test_installpath, '--debug', + '--disable-rpath', ] msg_regexs = [ diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 1c8c1771a6..918b344dcf 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -2269,6 +2269,7 @@ def test_compiler_cache(self): "--force", "--debug", "--disable-cleanup-tmpdir", + "--disable-rpath", ] ccache = which('ccache') From 92e9fe1ea235dec4b5859c9d6b27354bc763cc9e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 7 Feb 2024 18:11:35 +0100 Subject: [PATCH 295/312] fix broken tests after symlinking lib -> lib64 before postinstallcmds --- test/framework/toy_build.py | 42 +++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index f44182e2b8..f4ee8395c1 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2414,7 +2414,14 @@ def test_sanity_check_paths_lib64(self): # modify test easyconfig: move lib/libtoy.a to lib64/libtoy.a ectxt = re.sub(r"\s*'files'.*", "'files': ['bin/toy', ('lib/libtoy.a', 'lib/libfoo.a')],", ectxt) - postinstallcmd = "mkdir %(installdir)s/lib64 && mv %(installdir)s/lib/libtoy.a %(installdir)s/lib64/libtoy.a" + postinstallcmd = ' && '.join([ + # remove lib64 symlink (if it's there) + "rm -f %(installdir)s/lib64", + # create empty lib64 dir + "mkdir %(installdir)s/lib64", + # move libtoy.a + "mv %(installdir)s/lib/libtoy.a %(installdir)s/lib64/libtoy.a", + ]) ectxt = re.sub("postinstallcmds.*", "postinstallcmds = ['%s']" % postinstallcmd, ectxt) test_ec = os.path.join(self.test_prefix, 'toy-0.0.eb') @@ -3829,7 +3836,6 @@ def test_toy_build_lib_lib64_symlink(self): toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') test_ec_txt = read_file(toy_ec) - test_ec_txt += "\npostinstallcmds += ['mv %(installdir)s/lib %(installdir)s/lib64']" test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, test_ec_txt) @@ -3842,30 +3848,30 @@ def test_toy_build_lib_lib64_symlink(self): lib_path = os.path.join(toy_installdir, 'lib') lib64_path = os.path.join(toy_installdir, 'lib64') - # lib64 subdir exists, is not a symlink - self.assertExists(lib64_path) - self.assertTrue(os.path.isdir(lib64_path)) - self.assertFalse(os.path.islink(lib64_path)) - - # lib subdir is a symlink to lib64 subdir + # lib subdir exists, is not a symlink self.assertExists(lib_path) self.assertTrue(os.path.isdir(lib_path)) - self.assertTrue(os.path.islink(lib_path)) - self.assertTrue(os.path.samefile(lib_path, lib64_path)) + self.assertFalse(os.path.islink(lib_path)) - # lib symlink should point to a relative path - self.assertFalse(os.path.isabs(os.readlink(lib_path))) + # lib64 subdir is a symlink to lib subdir + self.assertExists(lib64_path) + self.assertTrue(os.path.isdir(lib64_path)) + self.assertTrue(os.path.islink(lib64_path)) + self.assertTrue(os.path.samefile(lib64_path, lib_path)) + + # lib64 symlink should point to a relative path + self.assertFalse(os.path.isabs(os.readlink(lib64_path))) # cleanup and try again with --disable-lib-lib64-symlink remove_dir(self.test_installpath) with self.mocked_stdout_stderr(): - self._test_toy_build(ec_file=test_ec, extra_args=['--disable-lib-lib64-symlink']) + self._test_toy_build(ec_file=test_ec, extra_args=['--disable-lib64-lib-symlink']) - self.assertExists(lib64_path) - self.assertNotExists(lib_path) - self.assertNotIn('lib', os.listdir(toy_installdir)) - self.assertTrue(os.path.isdir(lib64_path)) - self.assertFalse(os.path.islink(lib64_path)) + self.assertExists(lib_path) + self.assertNotExists(lib64_path) + self.assertNotIn('lib64', os.listdir(toy_installdir)) + self.assertTrue(os.path.isdir(lib_path)) + self.assertFalse(os.path.islink(lib_path)) def test_toy_build_sanity_check_linked_libs(self): """Test sanity checks for banned/requires libraries.""" From 612b1c0448db922adaed55c1f14ebb45f8bff2a8 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 7 Feb 2024 18:29:51 +0000 Subject: [PATCH 296/312] rpath off on mac --- easybuild/tools/options.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index cb6ec2022f..197514eb07 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -103,8 +103,8 @@ from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, OPTARCH_MAP_CHAR, OPTARCH_SEP, Compiler from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME from easybuild.tools.repository.repository import avail_repositories -from easybuild.tools.systemtools import UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family -from easybuild.tools.systemtools import get_cpu_features, get_gpu_info, get_system_info +from easybuild.tools.systemtools import DARWIN, UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family +from easybuild.tools.systemtools import get_cpu_features, get_gpu_info, get_os_type, get_system_info from easybuild.tools.version import this_is_easybuild @@ -131,6 +131,8 @@ def terminal_supports_colors(stream): DEFAULT_LIST_PR_ORDER = GITHUB_PR_ORDER_CREATED DEFAULT_LIST_PR_DIREC = GITHUB_PR_DIRECTION_DESC +RPATH_DEFAULT = False if get_os_type() == DARWIN else True + _log = fancylogger.getLogger('options', fname=False) @@ -489,7 +491,7 @@ def override_options(self): 'required-linked-shared-libs': ("Comma-separated list of shared libraries (names, file names, or paths) " "which must be linked in all installed binaries/libraries", 'strlist', 'extend', None), - 'rpath': ("Enable use of RPATH for linking with libraries", None, 'store_true', True), + 'rpath': ("Enable use of RPATH for linking with libraries", None, 'store_true', RPATH_DEFAULT), 'rpath-filter': ("List of regex patterns to use for filtering out RPATH paths", 'strlist', 'store', None), 'rpath-override-dirs': ("Path(s) to be prepended when linking with RPATH (string, colon-separated)", None, 'store', None), From 0960bc4f459826e6f378ad963ebe79d1b9474e8e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 8 Feb 2024 13:14:50 +0100 Subject: [PATCH 297/312] clean up log file of EasyBlock instance in check_sha256_checksums --- easybuild/framework/easyconfig/tools.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index c6649c383b..73393040c6 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -701,7 +701,9 @@ def check_sha256_checksums(ecs, whitelist=None): continue eb_class = get_easyblock_class(ec['easyblock'], name=ec['name']) - checksum_issues.extend(eb_class(ec).check_checksums()) + eb = eb_class(ec) + checksum_issues.extend(eb.check_checksums()) + eb.close_log() return checksum_issues From 22e12b4c068028a34ed2a85424b330bce96a090d Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 14 Feb 2024 15:43:45 +0000 Subject: [PATCH 298/312] allow only alphanumeric characters in the output filename --- easybuild/tools/run.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 92c92a1b41..e525f8f95a 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -42,6 +42,7 @@ import re import signal import shutil +import string import subprocess import sys import tempfile @@ -233,7 +234,6 @@ def to_cmd_str(cmd): work_dir = os.getcwd() cmd_str = to_cmd_str(cmd) - cmd_name = os.path.basename(cmd_str.split(' ')[0]) # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely if stream_output is None and build_option('logtostdout'): @@ -244,6 +244,9 @@ def to_cmd_str(cmd): if output_file: toptmpdir = os.path.join(tempfile.gettempdir(), 'run-shell-cmd-output') os.makedirs(toptmpdir, exist_ok=True) + # restrict the allowed characters in the name of the output_file + allowed_chars = string.ascii_letters + string.digits + cmd_name = ''.join([c for c in os.path.basename(cmd_str.split(' ')[0]) if c in allowed_chars]) tmpdir = tempfile.mkdtemp(dir=toptmpdir, prefix=f'{cmd_name}-') cmd_out_fp = os.path.join(tmpdir, 'out.txt') _log.info(f'run_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}') From faf8f1439b51f388fad600021a316df1acd9ab32 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Thu, 15 Feb 2024 11:14:33 +0000 Subject: [PATCH 299/312] functionalise the simplification and add tests --- easybuild/tools/run.py | 17 ++++++++++++++--- test/framework/run.py | 21 ++++++++++++++++++++- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index e525f8f95a..a2854ad8fb 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -181,6 +181,19 @@ def cache_aware_func(cmd, *args, **kwargs): run_shell_cmd_cache = run_cmd_cache +def fileprefix_from_cmd(cmd, allowed_chars=False): + """ + Simplify the cmd to only the allowed_chars we want in a filename + + :param cmd: the cmd (string) + :param allowed_chars: characters allowed in filename (defaults to string.ascii_letters + string.digits) + """ + if not allowed_chars: + allowed_chars = string.ascii_letters + string.digits + + return ''.join([c for c in cmd if c in allowed_chars]) + + @run_shell_cmd_cache def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, use_bash=True, @@ -244,9 +257,7 @@ def to_cmd_str(cmd): if output_file: toptmpdir = os.path.join(tempfile.gettempdir(), 'run-shell-cmd-output') os.makedirs(toptmpdir, exist_ok=True) - # restrict the allowed characters in the name of the output_file - allowed_chars = string.ascii_letters + string.digits - cmd_name = ''.join([c for c in os.path.basename(cmd_str.split(' ')[0]) if c in allowed_chars]) + cmd_name = fileprefix_from_cmd(os.path.basename(cmd_str.split(' ')[0])) tmpdir = tempfile.mkdtemp(dir=toptmpdir, prefix=f'{cmd_name}-') cmd_out_fp = os.path.join(tmpdir, 'out.txt') _log.info(f'run_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}') diff --git a/test/framework/run.py b/test/framework/run.py index db74940aec..22fd2a86ba 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -35,6 +35,7 @@ import os import re import signal +import string import stat import subprocess import sys @@ -51,7 +52,7 @@ from easybuild.tools.config import update_build_option from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir, read_file, write_file from easybuild.tools.run import RunShellCmdResult, RunShellCmdError, check_async_cmd, check_log_for_errors -from easybuild.tools.run import complete_cmd, get_output_from_process, parse_log_for_error +from easybuild.tools.run import complete_cmd, fileprefix_from_cmd, get_output_from_process, parse_log_for_error from easybuild.tools.run import run_cmd, run_cmd_qa, run_shell_cmd, subprocess_terminate from easybuild.tools.config import ERROR, IGNORE, WARN @@ -193,6 +194,24 @@ def test_run_shell_cmd_basic(self): self.assertTrue(isinstance(res.output, str)) self.assertTrue(res.work_dir and isinstance(res.work_dir, str)) + def test_fileprefix_from_cmd(self): + """test simplifications from fileprefix_from_cmd.""" + cmds = { + 'abd123': 'abd123', + 'ab"a': 'aba', + 'a{:$:S@"a': 'aSa', + } + for cmd, expected_simplification in cmds.items(): + self.assertEqual(fileprefix_from_cmd(cmd), expected_simplification) + + cmds = { + 'abd123': 'abd', + 'ab"a': 'aba', + '0a{:$:2@"a': 'aa', + } + for cmd, expected_simplification in cmds.items(): + self.assertEqual(fileprefix_from_cmd(cmd, allowed_chars=string.ascii_letters), expected_simplification) + def test_run_cmd_log(self): """Test logging of executed commands.""" fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') From 6c000d43c26a49f3b714e7faf6c1f0fa1ef253c6 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 17 Feb 2024 13:03:53 +0000 Subject: [PATCH 300/312] enhance download instructions --- easybuild/framework/easyblock.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a684b74347..aac45552e7 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -53,6 +53,7 @@ import time import traceback from datetime import datetime +from textwrap import indent import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain @@ -952,7 +953,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No if download_instructions is None: download_instructions = self.cfg['download_instructions'] if download_instructions is not None and download_instructions != "": - msg = "\nDownload instructions:\n\n" + download_instructions + '\n' + msg = "\nDownload instructions:\n\n" + indent(download_instructions, ' ') + '\n\n' + msg += "Make the files available in the active source path: %s\n" % ':'.join(source_paths()) print_msg(msg, prefix=False, stderr=True) error_msg += "please follow the download instructions above, and make the file available " error_msg += "in the active source path (%s)" % ':'.join(source_paths()) From 3e8632f7ab414ecbc52ac59eee6bcfb31133cdcb Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 17 Feb 2024 13:09:40 +0000 Subject: [PATCH 301/312] update test --- test/framework/easyblock.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 64c554ca89..93d9fed793 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1582,7 +1582,8 @@ def test_download_instructions(self): self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) stderr = self.get_stderr().strip() self.mock_stderr(False) - self.assertIn("Download instructions:\n\nManual download from example.com required", stderr) + self.assertIn("Download instructions:\n\n Manual download from example.com required", stderr) + self.assertIn("Make the files available in the active source path", stderr) # create dummy source file write_file(os.path.join(os.path.dirname(self.eb_file), 'software_with_missing_sources-0.0.tar.gz'), '') @@ -1596,7 +1597,8 @@ def test_download_instructions(self): stderr = self.get_stderr().strip() self.mock_stderr(False) self.mock_stdout(False) - self.assertIn("Download instructions:\n\nManual download from example.com required", stderr) + self.assertIn("Download instructions:\n\n Manual download from example.com required", stderr) + self.assertIn("Make the files available in the active source path", stderr) # wipe top-level download instructions, try again self.contents = self.contents.replace(download_instructions, '') @@ -1625,7 +1627,8 @@ def test_download_instructions(self): stderr = self.get_stderr().strip() self.mock_stderr(False) self.mock_stdout(False) - self.assertIn("Download instructions:\n\nExtension sources must be downloaded via example.com", stderr) + self.assertIn("Download instructions:\n\n Extension sources must be downloaded via example.com", stderr) + self.assertIn("Make the files available in the active source path", stderr) # download instructions should also be printed if 'source_tmpl' is used to specify extension sources self.contents = self.contents.replace(sources, "'source_tmpl': SOURCE_TAR_GZ,") @@ -1638,7 +1641,8 @@ def test_download_instructions(self): stderr = self.get_stderr().strip() self.mock_stderr(False) self.mock_stdout(False) - self.assertIn("Download instructions:\n\nExtension sources must be downloaded via example.com", stderr) + self.assertIn("Download instructions:\n\n Extension sources must be downloaded via example.com", stderr) + self.assertIn("Make the files available in the active source path", stderr) # create dummy source file for extension write_file(os.path.join(os.path.dirname(self.eb_file), 'ext_with_missing_sources-0.0.tar.gz'), '') From ca3828bc1c1a16f29287e62b11c6a705aa86aceb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 21 Feb 2024 14:34:09 +0100 Subject: [PATCH 302/312] refactor EasyBlock.skip_extensions_parallel --- easybuild/framework/easyblock.py | 33 ++++++++++---------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index e2d84c162e..64aa182dc7 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -41,6 +41,7 @@ * Davide Vanzo (Vanderbilt University) * Caspar van Leeuwen (SURF) """ +import concurrent import copy import glob import inspect @@ -1818,22 +1819,20 @@ def skip_extensions_parallel(self, exts_filter): self.log.experimental("Skipping installed extensions in parallel") print_msg("skipping installed extensions (in parallel)", log=self.log) - thread_pool = ThreadPoolExecutor(max_workers=self.cfg['parallel']) - - async_shell_cmd_tasks = [] installed_exts_ids = [] - exts_queue = list(enumerate(self.ext_instances[:])) checked_exts_cnt = 0 exts_cnt = len(self.ext_instances) - done_tasks = [] + cmds = [resolve_exts_filter_template(exts_filter, ext) for ext in self.ext_instances] - # asynchronously run checks to see whether extensions are already installed - while exts_queue or async_shell_cmd_tasks: + with ThreadPoolExecutor(max_workers=self.cfg['parallel']) as thread_pool: - # first handle completed checks - for task in done_tasks: - async_shell_cmd_tasks.remove(task) - res = task.result() + # list of command to run asynchronously + async_cmds = [thread_pool.submit(run_shell_cmd, cmd, stdin=stdin, hidden=True, fail_on_error=False, + asynchronous=True, task_id=idx) for (idx, (cmd, stdin)) in enumerate(cmds)] + + # process result of commands as they have completed running + for done_task in concurrent.futures.as_completed(async_cmds): + res = done_task.result() idx = res.task_id ext_name = self.ext_instances[idx].name self.log.info(f"exts_filter result for {ext_name}: exit code {res.exit_code}; output: {res.output}") @@ -1846,16 +1845,6 @@ def skip_extensions_parallel(self, exts_filter): exts_pbar_label += "(%d/%d checked)" % (checked_exts_cnt, exts_cnt) self.update_exts_progress_bar(exts_pbar_label) - # start additional checks asynchronously - while exts_queue: - idx, ext = exts_queue.pop(0) - cmd, stdin = resolve_exts_filter_template(exts_filter, ext) - task = thread_pool.submit(run_shell_cmd, cmd, stdin=stdin, hidden=True, - fail_on_error=False, asynchronous=True, task_id=idx) - async_shell_cmd_tasks.append(task) - - (done_tasks, _) = wait(async_shell_cmd_tasks, timeout=1, return_when=FIRST_COMPLETED) - # compose new list of extensions, skip over the ones that are already installed; # note: original order in extensions list should be preserved! retained_ext_instances = [] @@ -1866,8 +1855,6 @@ def skip_extensions_parallel(self, exts_filter): self.ext_instances = retained_ext_instances - thread_pool.shutdown() - def install_extensions(self, install=True): """ Install extensions. From f7c0ff2295789736557861ede1b82172e40f6ab7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 21 Feb 2024 15:51:16 +0100 Subject: [PATCH 303/312] clean up unused imports from concurrent.futures --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 64aa182dc7..4dd4bcfd81 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -52,7 +52,7 @@ import tempfile import time import traceback -from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait +from concurrent.futures import ThreadPoolExecutor from datetime import datetime import easybuild.tools.environment as env From cb84bd2ba1610419e03f869053d3bd158e5ada1a Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 22 Feb 2024 15:08:58 +0000 Subject: [PATCH 304/312] extra cases --- easybuild/tools/run.py | 4 ++-- test/framework/run.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index a2854ad8fb..6a568a9f70 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -186,10 +186,10 @@ def fileprefix_from_cmd(cmd, allowed_chars=False): Simplify the cmd to only the allowed_chars we want in a filename :param cmd: the cmd (string) - :param allowed_chars: characters allowed in filename (defaults to string.ascii_letters + string.digits) + :param allowed_chars: characters allowed in filename (defaults to string.ascii_letters + string.digits + "_-") """ if not allowed_chars: - allowed_chars = string.ascii_letters + string.digits + allowed_chars = f"{string.ascii_letters}{string.digits}_-" return ''.join([c for c in cmd if c in allowed_chars]) diff --git a/test/framework/run.py b/test/framework/run.py index 22fd2a86ba..386a9cf7f6 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -200,6 +200,8 @@ def test_fileprefix_from_cmd(self): 'abd123': 'abd123', 'ab"a': 'aba', 'a{:$:S@"a': 'aSa', + 'cmd-with-dash': 'cmd-with-dash', + 'cmd_with_underscore'. 'cmd_with_underscore', } for cmd, expected_simplification in cmds.items(): self.assertEqual(fileprefix_from_cmd(cmd), expected_simplification) From a309aaea65a335abee90f4a8df09907c92eaed20 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 22 Feb 2024 15:09:40 +0000 Subject: [PATCH 305/312] typo --- test/framework/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/run.py b/test/framework/run.py index 386a9cf7f6..e63e4e219c 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -201,7 +201,7 @@ def test_fileprefix_from_cmd(self): 'ab"a': 'aba', 'a{:$:S@"a': 'aSa', 'cmd-with-dash': 'cmd-with-dash', - 'cmd_with_underscore'. 'cmd_with_underscore', + 'cmd_with_underscore': 'cmd_with_underscore', } for cmd, expected_simplification in cmds.items(): self.assertEqual(fileprefix_from_cmd(cmd), expected_simplification) From 560d395a1d8ab4ef2366543b642d15ec548c1c1a Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Thu, 22 Feb 2024 16:13:41 +0000 Subject: [PATCH 306/312] re-order test_cuda_compute_capabilities {pre,}{config,build,install}opts --- test/framework/easyconfig.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 11b551e55c..1ce8d1facc 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4438,31 +4438,31 @@ def test_cuda_compute_capabilities(self): description = 'test' toolchain = SYSTEM cuda_compute_capabilities = ['5.1', '7.0', '7.1'] - installopts = '%(cuda_compute_capabilities)s' - preinstallopts = '%(cuda_cc_space_sep)s' - prebuildopts = '%(cuda_cc_semicolon_sep)s' - configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"' preconfigopts = 'CUDAARCHS="%(cuda_cc_cmake)s"' + configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"' + prebuildopts = '%(cuda_cc_semicolon_sep)s' + preinstallopts = '%(cuda_cc_space_sep)s' + installopts = '%(cuda_compute_capabilities)s' """) self.prep() ec = EasyConfig(self.eb_file) - self.assertEqual(ec['installopts'], '5.1,7.0,7.1') - self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1') - self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1') + self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="51;70;71"') self.assertEqual(ec['configopts'], 'comma="sm_51,sm_70,sm_71" ' 'space="sm_51 sm_70 sm_71"') - self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="51;70;71"') + self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1') + self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1') + self.assertEqual(ec['installopts'], '5.1,7.0,7.1') # build options overwrite it init_config(build_options={'cuda_compute_capabilities': ['4.2', '6.3']}) ec = EasyConfig(self.eb_file) - self.assertEqual(ec['installopts'], '4.2,6.3') - self.assertEqual(ec['preinstallopts'], '4.2 6.3') - self.assertEqual(ec['prebuildopts'], '4.2;6.3') + self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="42;63"') self.assertEqual(ec['configopts'], 'comma="sm_42,sm_63" ' 'space="sm_42 sm_63"') - self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="42;63"') + self.assertEqual(ec['prebuildopts'], '4.2;6.3') + self.assertEqual(ec['preinstallopts'], '4.2 6.3') + self.assertEqual(ec['installopts'], '4.2,6.3') def test_det_copy_ec_specs(self): """Test det_copy_ec_specs function.""" From aff74821c1cc9b3e69065d370e4bd64318162a26 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Thu, 22 Feb 2024 16:37:45 +0000 Subject: [PATCH 307/312] add support for cuda compute capabilities templates in the form of: cuda_int_comma_sep: 70,75 80 cuda_int_semicolon_sep: 70;75;80 cuda_int_space_sep: 70 75 80 --- easybuild/framework/easyconfig/templates.py | 7 +++++++ test/framework/easyconfig.py | 11 +++++++++++ 2 files changed, 18 insertions(+) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index fcefe4de55..0d8a4d5cf3 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -97,6 +97,9 @@ ('cuda_cc_cmake', "List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+"), ('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"), ('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"), + ('cuda_int_comma_sep', "Comma-separated list of integer CUDA compute capabilities"), + ('cuda_int_space_sep', "Space-separated list of integer CUDA compute capabilities"), + ('cuda_int_semicolon_sep', "Semicolon-separated list of integer CUDA compute capabilities"), ('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"), ('cuda_sm_space_sep', "Space-separated list of sm_* values that correspond with CUDA compute capabilities"), ] @@ -365,6 +368,10 @@ def template_constant_dict(config, ignore=None, toolchain=None): template_values['cuda_cc_space_sep'] = ' '.join(cuda_compute_capabilities) template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_compute_capabilities) template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_compute_capabilities) + int_values = [cc.replace('.', '') for cc in cuda_compute_capabilities] + template_values['cuda_int_comma_sep'] = ','.join(int_values) + template_values['cuda_int_space_sep'] = ' '.join(int_values) + template_values['cuda_int_semicolon_sep'] = ';'.join(int_values) sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_compute_capabilities] template_values['cuda_sm_comma_sep'] = ','.join(sm_values) template_values['cuda_sm_space_sep'] = ' '.join(sm_values) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 1ce8d1facc..399ce3afe8 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4441,6 +4441,8 @@ def test_cuda_compute_capabilities(self): preconfigopts = 'CUDAARCHS="%(cuda_cc_cmake)s"' configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"' prebuildopts = '%(cuda_cc_semicolon_sep)s' + buildopts = ('comma="%(cuda_int_comma_sep)s" space="%(cuda_int_space_sep)s" + 'semi="%(cuda_int_semicolon_sep)s"') preinstallopts = '%(cuda_cc_space_sep)s' installopts = '%(cuda_compute_capabilities)s' """) @@ -4451,6 +4453,9 @@ def test_cuda_compute_capabilities(self): self.assertEqual(ec['configopts'], 'comma="sm_51,sm_70,sm_71" ' 'space="sm_51 sm_70 sm_71"') self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1') + self.assertEqual(ec['buildopts'], 'comma="51,70,71" ' + 'space="51 70 71" ' + 'semi="51;70;71"') self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1') self.assertEqual(ec['installopts'], '5.1,7.0,7.1') @@ -4460,6 +4465,9 @@ def test_cuda_compute_capabilities(self): self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="42;63"') self.assertEqual(ec['configopts'], 'comma="sm_42,sm_63" ' 'space="sm_42 sm_63"') + self.assertEqual(ec['buildopts'], 'comma="42,63" ' + 'space="42 63" ' + 'semi="42;63"') self.assertEqual(ec['prebuildopts'], '4.2;6.3') self.assertEqual(ec['preinstallopts'], '4.2 6.3') self.assertEqual(ec['installopts'], '4.2,6.3') @@ -4725,6 +4733,9 @@ def test_get_cuda_cc_template_value(self): 'cuda_compute_capabilities': '6.5,7.0', 'cuda_cc_space_sep': '6.5 7.0', 'cuda_cc_semicolon_sep': '6.5;7.0', + 'cuda_int_comma_sep': '65,70', + 'cuda_int_space_sep': '65 70', + 'cuda_int_semicolon_sep': '65;70', 'cuda_sm_comma_sep': 'sm_65,sm_70', 'cuda_sm_space_sep': 'sm_65 sm_70', } From f2d53d1ca449c404a246fe54bc9be897758ef39f Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Thu, 22 Feb 2024 17:03:12 +0000 Subject: [PATCH 308/312] fix missing end quote --- test/framework/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 399ce3afe8..a61be39473 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4441,7 +4441,7 @@ def test_cuda_compute_capabilities(self): preconfigopts = 'CUDAARCHS="%(cuda_cc_cmake)s"' configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"' prebuildopts = '%(cuda_cc_semicolon_sep)s' - buildopts = ('comma="%(cuda_int_comma_sep)s" space="%(cuda_int_space_sep)s" + buildopts = ('comma="%(cuda_int_comma_sep)s" space="%(cuda_int_space_sep)s"' 'semi="%(cuda_int_semicolon_sep)s"') preinstallopts = '%(cuda_cc_space_sep)s' installopts = '%(cuda_compute_capabilities)s' From b65771f3132a0dc3b9a9fae6f3b015215637ba62 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Thu, 22 Feb 2024 17:41:50 +0000 Subject: [PATCH 309/312] fix missing space --- test/framework/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index a61be39473..141e9b0f77 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4441,7 +4441,7 @@ def test_cuda_compute_capabilities(self): preconfigopts = 'CUDAARCHS="%(cuda_cc_cmake)s"' configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"' prebuildopts = '%(cuda_cc_semicolon_sep)s' - buildopts = ('comma="%(cuda_int_comma_sep)s" space="%(cuda_int_space_sep)s"' + buildopts = ('comma="%(cuda_int_comma_sep)s" space="%(cuda_int_space_sep)s" ' 'semi="%(cuda_int_semicolon_sep)s"') preinstallopts = '%(cuda_cc_space_sep)s' installopts = '%(cuda_compute_capabilities)s' From 8d9e14eb6d72490c675adb8a70415a4c735cc0bc Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 29 Feb 2024 02:40:57 +0100 Subject: [PATCH 310/312] make reproducible archives only of git repos without .git dir and reset timestamps with touch --- easybuild/tools/filetools.py | 42 ++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 12fe557a60..5b7406fffa 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2540,12 +2540,12 @@ def copy(paths, target_path, force_in_dry_run=False, **kwargs): raise EasyBuildError("Specified path to copy is not an existing file or directory: %s", path) -def get_source_tarball_from_git(filename, targetdir, git_config): +def get_source_tarball_from_git(filename, target_dir, git_config): """ Downloads a git repository, at a specific tag or commit, recursively or not, and make an archive with it :param filename: name of the archive to save the code to (must be .tar.gz) - :param targetdir: target directory where to save the archive to + :param target_dir: target directory where to save the archive to :param git_config: dictionary containing url, repo_name, recursive, and one of tag or commit """ # sanity check on git_config value being passed @@ -2584,8 +2584,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): raise EasyBuildError("git_config currently only supports filename ending in .tar.gz") # prepare target directory and clone repository - mkdir(targetdir, parents=True) - targetpath = os.path.join(targetdir, filename) + mkdir(target_dir, parents=True) # compose 'git clone' command, and run it if extra_config_params: @@ -2668,21 +2667,36 @@ def get_source_tarball_from_git(filename, targetdir, git_config): for cmd in cmds: run_shell_cmd(cmd, work_dir=work_dir, hidden=True, verbose_dry_run=True) - # When CentOS 7 is phased out and tar>1.28 is everywhere, replace find-sort-pipe with tar-flag - # '--sort=name' and place LC_ALL in front of tar. Also remove flags --null, --no-recursion, and - # --files-from - from the flags to tar. See https://reproducible-builds.org/docs/archives/ - tar_cmd = ['find', repo_name, '-print0', '-path \'*/.git\' -prune' if not keep_git_dir else '', '|', - 'LC_ALL=C', 'sort', '--zero-terminated', '|', - 'GZIP=--no-name', 'tar', '--create', '--file', targetpath, '--no-recursion', - '--gzip', '--mtime="1970-01-01 00:00Z"', '--owner=0', '--group=0', - '--numeric-owner', '--format=gnu', '--null', - '--no-recursion', '--files-from -'] + # Create archive + archive_path = os.path.join(target_dir, filename) + + if keep_git_dir: + # create archive of git repo including .git directory + tar_cmd = ['tar', 'cfvz', archive_path, repo_name] + else: + # create reproducible archive + # see https://reproducible-builds.org/docs/archives/ + # TODO: when CentOS 7 is phased out and tar>1.28 is everywhere, replace sort step + # in the pipe with tar-flag '--sort=name' and place LC_ALL in front of tar. + tar_cmd = [ + # print names of all files and folders excluding .git directory + 'find', repo_name, '-name ".git"', '-prune', '-o', '-print0', + # reset access and modification timestamps + '-exec', 'touch', '-t 197001010100', '{}', '\;', '|', + # sort file list + 'LC_ALL=C', 'sort', '--zero-terminated', '|', + # create tarball in GNU format with ownership reset + 'tar', '--create', '--no-recursion', '--owner=0', '--group=0', '--numeric-owner', '--format=gnu', + '--null', '--files-from', '-', '|', + # compress tarball with gzip without original file name and timestamp + 'gzip', '--no-name', '>', archive_path + ] run_shell_cmd(' '.join(tar_cmd), work_dir=tmpdir, hidden=True, verbose_dry_run=True) # cleanup (repo_name dir does not exist in dry run mode) remove(tmpdir) - return targetpath + return archive_path def move_file(path, target_path, force_in_dry_run=False): From 46e61c85d881dc081936347930cfa44bdbd42bde Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 29 Feb 2024 02:52:25 +0100 Subject: [PATCH 311/312] update FileToolsTest.test_github_get_source_tarball_from_git tests with reproducible tar commands --- test/framework/filetools.py | 63 +++++++++++++++---------------------- 1 file changed, 25 insertions(+), 38 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 3adb736586..8d823ef59c 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2802,26 +2802,25 @@ def run_check(): 'git_repo': 'git@github.com:easybuilders/testrepository.git', 'test_prefix': self.test_prefix, } + reprod_tar_cmd_pattern = ( + r' running shell command "find {} -name \".git\" -prune -o -print0 -exec touch -t 197001010100 {{}} \; |' + r' LC_ALL=C sort --zero-terminated | tar --create --no-recursion --owner=0 --group=0 --numeric-owner' + r' --format=gnu --null --files-from - | gzip --no-name > %(test_prefix)s/target/test.tar.gz' + ) expected = '\n'.join([ - r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"', + r' running shell command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' - r' | GZIP=--no-name tar --create --file %(test_prefix)s/target/test.tar.gz --no-recursion' - r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' - r' --null --no-recursion --files-from -"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in .*/tmp.*\)", ]) % string_args run_check() git_config['clone_into'] = 'test123' expected = '\n'.join([ - r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"', + r' running shell command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"', r" \(in .*/tmp.*\)", - r' running command "find test123 -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' - r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' - r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' - r' --null --no-recursion --files-from -"', + reprod_tar_cmd_pattern.format("test123"), r" \(in .*/tmp.*\)", ]) % string_args run_check() @@ -2829,12 +2828,9 @@ def run_check(): git_config['recursive'] = True expected = '\n'.join([ - r' running command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"', + r' running shell command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' - r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' - r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' - r' --null --no-recursion --files-from -"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in .*/tmp.*\)", ]) % string_args run_check() @@ -2844,9 +2840,9 @@ def run_check(): ' running shell command "git clone --depth 1 --branch tag_for_tests --recursive' + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in .*/tmp.*\)", - ]) % git_repo + ]) % string_args run_check() git_config['extra_config_params'] = [ @@ -2858,21 +2854,18 @@ def run_check(): + ' clone --depth 1 --branch tag_for_tests --recursive' + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in .*/tmp.*\)", - ]) % git_repo + ]) % string_args run_check() del git_config['recurse_submodules'] del git_config['extra_config_params'] git_config['keep_git_dir'] = True expected = '\n'.join([ - r' running command "git clone --branch tag_for_tests --recursive %(git_repo)s"', + r' running shell command "git clone --branch tag_for_tests --recursive %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "find testrepository -print0 | LC_ALL=C sort --zero-terminated | GZIP=--no-name tar' - r' --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion --gzip' - r' --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu --null --no-recursion' - r' --files-from -"', + r' running shell command "tar cfvz .*/target/test.tar.gz testrepository"', r" \(in .*/tmp.*\)", ]) % string_args run_check() @@ -2881,28 +2874,22 @@ def run_check(): del git_config['tag'] git_config['commit'] = '8456f86' expected = '\n'.join([ - r' running command "git clone --no-checkout %(git_repo)s"', + r' running shell command "git clone --no-checkout %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "git checkout 8456f86 && git submodule update --init --recursive"', + r' running shell command "git checkout 8456f86 && git submodule update --init --recursive"', r" \(in testrepository\)", - r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' - r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' - r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' - r' --null --no-recursion --files-from -"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in .*/tmp.*\)", ]) % string_args run_check() git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite'] expected = '\n'.join([ - r' running command "git clone --no-checkout %(git_repo)s"', + r' running shell command "git clone --no-checkout %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "git checkout 8456f86"', + r' running shell command "git checkout 8456f86"', r" \(in testrepository\)", - r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' - r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' - r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' - r' --null --no-recursion --files-from -"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in .*/tmp.*\)", ]) % string_args run_check() @@ -2914,9 +2901,9 @@ def run_check(): r" \(in /.*\)", r' running shell command "git checkout 8456f86"', r" \(in /.*/testrepository\)", - r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in /.*\)", - ]) % git_repo + ]) % string_args run_check() # Test with real data. From 8c76561bf026831bfb7ff065a923636f20dd4b1f Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 29 Feb 2024 09:56:49 +0100 Subject: [PATCH 312/312] remove TODO about sort option in tar as it is not supported across implementations --- easybuild/tools/filetools.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 5b7406fffa..470e347e88 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2676,8 +2676,6 @@ def get_source_tarball_from_git(filename, target_dir, git_config): else: # create reproducible archive # see https://reproducible-builds.org/docs/archives/ - # TODO: when CentOS 7 is phased out and tar>1.28 is everywhere, replace sort step - # in the pipe with tar-flag '--sort=name' and place LC_ALL in front of tar. tar_cmd = [ # print names of all files and folders excluding .git directory 'find', repo_name, '-name ".git"', '-prune', '-o', '-print0',