From fda5ca8fda5a16802dec3366242a72b07d2e7c96 Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 4 Oct 2024 16:00:54 +0200 Subject: [PATCH 1/6] Added possibility to call `amend/try-amend` multiple times from easystack --- easybuild/tools/options.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index fd22e384e5..9afb9629f5 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -2002,6 +2002,7 @@ def opts_dict_to_eb_opts(args_dict): :return: a list of strings representing command-line options for the 'eb' command """ + allow_multiple_calls = ['amend', 'try-amend'] _log.debug("Converting dictionary %s to argument list" % args_dict) args = [] for arg in sorted(args_dict): @@ -2011,6 +2012,14 @@ def opts_dict_to_eb_opts(args_dict): prefix = '--' option = prefix + str(arg) value = args_dict[arg] + + if str(arg) in allow_multiple_calls: + if not isinstance(value, (list, tuple)): + value = [value] + for v in value: + args.append(option + '=' + str(v)) + continue + if isinstance(value, (list, tuple)): value = ','.join(str(x) for x in value) From 550d96cb8aeeab15644d995d8dc9f4c1ca5f549f Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 4 Oct 2024 16:50:17 +0200 Subject: [PATCH 2/6] Refactor to remove `continue` --- easybuild/tools/options.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 9afb9629f5..6c7c7096b0 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -2018,16 +2018,13 @@ def opts_dict_to_eb_opts(args_dict): value = [value] for v in value: args.append(option + '=' + str(v)) - continue - - if isinstance(value, (list, tuple)): - value = ','.join(str(x) for x in value) - - if value in [True, None]: + elif value in [True, None]: args.append(option) elif value is False: args.append('--disable-' + option[2:]) elif value is not None: + if isinstance(value, (list, tuple)): + value = ','.join(str(x) for x in value) args.append(option + '=' + str(value)) _log.debug("Converted dictionary %s to argument list %s" % (args_dict, args)) From 72e21762fb70ee1c3a6cae1224fb859d178943b7 Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 4 Oct 2024 16:50:24 +0200 Subject: [PATCH 3/6] Added test case --- test/framework/options.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index 2208465db0..0b5f285464 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -7252,6 +7252,15 @@ def test_opts_dict_to_eb_opts(self): ] self.assertEqual(opts_dict_to_eb_opts(opts_dict), expected) + # multi-call options + opts_dict = {'try-amend': ['a=1', 'b=2', 'c=3']} + expected = ['--try-amend=a=1', '--try-amend=b=2', '--try-amend=c=3'] + self.assertEqual(opts_dict_to_eb_opts(opts_dict), expected) + + opts_dict = {'amend': ['a=1', 'b=2', 'c=3']} + expected = ['--amend=a=1', '--amend=b=2', '--amend=c=3'] + self.assertEqual(opts_dict_to_eb_opts(opts_dict), expected) + def suite(): """ returns all the testcases in this module """ From 201aba6e4dee8c710a9ca58e800524ce464eb1cb Mon Sep 17 00:00:00 2001 From: Davide Grassano <34096612+Crivella@users.noreply.github.com> Date: Fri, 4 Oct 2024 22:08:35 +0200 Subject: [PATCH 4/6] Change to inline Co-authored-by: Kenneth Hoste --- easybuild/tools/options.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 6c7c7096b0..cb65498a81 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -2016,8 +2016,7 @@ def opts_dict_to_eb_opts(args_dict): if str(arg) in allow_multiple_calls: if not isinstance(value, (list, tuple)): value = [value] - for v in value: - args.append(option + '=' + str(v)) + args.extend(option + '=' + str(x) for x in value) elif value in [True, None]: args.append(option) elif value is False: From b01a52b114156f1f5480083c8c0a38ae29674793 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 15 Oct 2024 10:55:26 +0200 Subject: [PATCH 5/6] Update findPythonDeps - Require Python 3.3 - Add/update documentation - Use f-strings for formatted output - Fix detecting packages where the name might appear as the "key" not the "package_name" - Set PYTHONNOUSERSITE - Use venv module instead of virtualenv to require less packages --- easybuild/scripts/findPythonDeps.py | 204 ++++++++++++++++------------ 1 file changed, 117 insertions(+), 87 deletions(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index 23d2763148..6b57b57ebb 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -1,5 +1,21 @@ #!/usr/bin/env python +""" +Find Python dependencies for a given Python package after loading dependencies specified in an EasyConfig. +This is intended for writing or updating PythonBundle EasyConfigs: + 1. Create a EasyConfig with at least 'Python' as a dependency. + When updating to a new toolchain it is a good idea to reduce the dependencies to a minimum + as e.g. the new "Python" module might have different packages included. + 2. Run this script + 3. For each dependency found by this script search existing EasyConfigs for ones providing that Python package. + E.g many are contained in Python-bundle-PyPI. Some can be updated from an earlier toolchain. + 4. Add those EasyConfigs as dependencies to your new EasyConfig. + 5. Rerun this script so it takes the newly provided packages into account. + You can do steps 3-5 iteratively adding EasyConfig-dependencies one-by-one. + 6. Finally you copy the packages found by this script as "exts_list" into the new EasyConfig. + You usually want the list printed as "in install order", the format is already suitable to be copied as-is. +""" + import argparse import json import os @@ -8,12 +24,13 @@ import subprocess import sys import tempfile +import textwrap from contextlib import contextmanager from pprint import pprint try: import pkg_resources except ImportError as e: - print('pkg_resources could not be imported: %s\nYou might need to install setuptools!' % e) + print(f'pkg_resources could not be imported: {e}\nYou might need to install setuptools!') sys.exit(1) try: @@ -22,6 +39,7 @@ _canonicalize_regex = re.compile(r"[-_.]+") def canonicalize_name(name): + """Fallback if the import doesn't work with same behavior.""" return _canonicalize_regex.sub("-", name).lower() @@ -36,16 +54,16 @@ def temporary_directory(*args, **kwargs): def extract_pkg_name(package_spec): - return re.split('<|>|=|~', args.package, 1)[0] + """Get the package name from a specification such as 'package>=3.42'""" + return re.split('<|>|=|~', package_spec, 1)[0] -def can_run(cmd, argument): +def can_run(cmd, *arguments): """Check if the given cmd and argument can be run successfully""" - with open(os.devnull, 'w') as FNULL: - try: - return subprocess.call([cmd, argument], stdout=FNULL, stderr=subprocess.STDOUT) == 0 - except (subprocess.CalledProcessError, OSError): - return False + try: + return subprocess.call([cmd, *arguments], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) == 0 + except (subprocess.CalledProcessError, OSError): + return False def run_cmd(arguments, action_desc, capture_stderr=True, **kwargs): @@ -59,13 +77,13 @@ def run_cmd(arguments, action_desc, capture_stderr=True, **kwargs): if p.returncode != 0: if err: err = "\nSTDERR:\n" + err - raise RuntimeError('Failed to %s: %s%s' % (action_desc, out, err)) + raise RuntimeError(f'Failed to {action_desc}: {out}{err}') return out def run_in_venv(cmd, venv_path, action_desc): """Run the given command in the virtualenv at the given path""" - cmd = 'source %s/bin/activate && %s' % (venv_path, cmd) + cmd = f'source {venv_path}/bin/activate && {cmd}' return run_cmd(cmd, action_desc, shell=True, executable='/bin/bash') @@ -83,12 +101,12 @@ def get_dep_tree(package_spec, verbose): print('Updating pip in virtualenv') run_in_venv('pip install --upgrade pip', venv_dir, action_desc='update pip') if verbose: - print('Installing %s into virtualenv' % package_spec) - out = run_in_venv('pip install "%s"' % package_spec, venv_dir, action_desc='install ' + package_spec) - print('%s installed: %s' % (package_spec, out)) + print(f'Installing {package_spec} into virtualenv') + out = run_in_venv(f'pip install "{package_spec}"', venv_dir, action_desc='install ' + package_spec) + print(f'{package_spec} installed: {out}') # install pipdeptree, figure out dependency tree for installed package run_in_venv('pip install pipdeptree', venv_dir, action_desc='install pipdeptree') - dep_tree = run_in_venv('pipdeptree -j -p "%s"' % package_name, + dep_tree = run_in_venv(f'pipdeptree -j -p "{package_name}"', venv_dir, action_desc='collect dependencies') return json.loads(dep_tree) @@ -108,7 +126,7 @@ def find_deps(pkgs, dep_tree): for orig_pkg in cur_pkgs: count += 1 if count > MAX_PACKAGES: - raise RuntimeError("Aborting after checking %s packages. Possibly cycle detected!" % MAX_PACKAGES) + raise RuntimeError(f"Aborting after checking {MAX_PACKAGES} packages. Possibly cycle detected!") pkg = canonicalize_name(orig_pkg) matching_entries = [entry for entry in dep_tree if pkg in (entry['package']['package_name'], entry['package']['key'])] @@ -116,11 +134,11 @@ def find_deps(pkgs, dep_tree): matching_entries = [entry for entry in dep_tree if orig_pkg in (entry['package']['package_name'], entry['package']['key'])] if not matching_entries: - raise RuntimeError("Found no installed package for '%s' in %s" % (pkg, dep_tree)) + raise RuntimeError(f"Found no installed package for '{pkg}' in {dep_tree}") if len(matching_entries) > 1: - raise RuntimeError("Found multiple installed packages for '%s' in %s" % (pkg, dep_tree)) + raise RuntimeError(f"Found multiple installed packages for '{pkg}' in {dep_tree}") entry = matching_entries[0] - res.append((entry['package']['package_name'], entry['package']['installed_version'])) + res.append(entry['package']) # Add dependencies to list of packages to check next # Could call this function recursively but that might exceed the max recursion depth next_pkgs.update(dep['package_name'] for dep in entry['dependencies']) @@ -128,6 +146,7 @@ def find_deps(pkgs, dep_tree): def print_deps(package, verbose): + """Print dependencies of the given package that are not installed yet in a format usable as 'exts_list'""" if verbose: print('Getting dep tree of ' + package) dep_tree = get_dep_tree(package, verbose) @@ -144,13 +163,18 @@ def print_deps(package, verbose): res = [] handled = set() for dep in reversed(deps): - if dep not in handled: - handled.add(dep) - if dep[0] in installed_modules: + # Tuple as we need it for exts_list + dep_entry = (dep['package_name'], dep['installed_version']) + if dep_entry not in handled: + handled.add(dep_entry) + # Need to check for key and package_name as naming is not consistent. E.g.: + # "PyQt5-sip": 'key': 'pyqt5-sip', 'package_name': 'PyQt5-sip' + # "jupyter-core": 'key': 'jupyter-core', 'package_name': 'jupyter_core' + if dep['key'] in installed_modules or dep['package_name'] in installed_modules: if verbose: - print("Skipping installed module '%s'" % dep[0]) + print(f"Skipping installed module '{dep['package_name']}'") else: - res.append(dep) + res.append(dep_entry) print("List of dependencies in (likely) install order:") pprint(res, indent=4) @@ -158,69 +182,75 @@ def print_deps(package, verbose): pprint(sorted(res), indent=4) -examples = [ - 'Example usage with EasyBuild (after installing dependency modules):', - '\t' + sys.argv[0] + ' --ec TensorFlow-2.3.4.eb tensorflow==2.3.4', - 'Which is the same as:', - '\t' + ' && '.join(['eb TensorFlow-2.3.4.eb --dump-env', - 'source TensorFlow-2.3.4.env', - sys.argv[0] + ' tensorflow==2.3.4', - ]), -] -parser = argparse.ArgumentParser( - description='Find dependencies of Python packages by installing it in a temporary virtualenv. ', - epilog='\n'.join(examples), - formatter_class=argparse.RawDescriptionHelpFormatter -) -parser.add_argument('package', metavar='python-pkg-spec', - help='Python package spec, e.g. tensorflow==2.3.4') -parser.add_argument('--ec', metavar='easyconfig', help='EasyConfig to use as the build environment. ' - 'You need to have dependency modules installed already!') -parser.add_argument('--verbose', help='Verbose output', action='store_true') -args = parser.parse_args() - -if args.ec: - if not can_run('eb', '--version'): - print('EasyBuild not found or executable. Make sure it is in your $PATH when using --ec!') - sys.exit(1) - if args.verbose: - print('Checking with EasyBuild for missing dependencies') - missing_dep_out = run_cmd(['eb', args.ec, '--missing'], - capture_stderr=False, - action_desc='Get missing dependencies' - ) - excluded_dep = '(%s)' % os.path.basename(args.ec) - missing_deps = [dep for dep in missing_dep_out.split('\n') - if dep.startswith('*') and excluded_dep not in dep - ] - if missing_deps: - print('You need to install all modules on which %s depends first!' % args.ec) - print('\n\t'.join(['Missing:'] + missing_deps)) - sys.exit(1) - - # If the --ec argument is a (relative) existing path make it absolute so we can find it after the chdir - ec_arg = os.path.abspath(args.ec) if os.path.exists(args.ec) else args.ec - with temporary_directory() as tmp_dir: - old_dir = os.getcwd() - os.chdir(tmp_dir) - if args.verbose: - print('Running EasyBuild to get build environment') - run_cmd(['eb', ec_arg, '--dump-env', '--force'], action_desc='Dump build environment') - os.chdir(old_dir) +def main(): + """Entrypoint of the script""" + examples = textwrap.dedent(f""" + Example usage with EasyBuild (after installing dependency modules): + {sys.argv[0]} --ec TensorFlow-2.3.4.eb tensorflow==2.3.4 + Which is the same as: + eb TensorFlow-2.3.4.eb --dump-env && source TensorFlow-2.3.4.env && {sys.argv[0]} tensorflow==2.3.4 + Using the '--ec' parameter is recommended as the latter requires manually updating the .env file + after each change to the EasyConfig. + """) + parser = argparse.ArgumentParser( + description='Find dependencies of Python packages by installing it in a temporary virtualenv. ', + epilog='\n'.join(examples), + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument('package', metavar='python-pkg-spec', + help='Python package spec, e.g. tensorflow==2.3.4') + parser.add_argument('--ec', metavar='easyconfig', help='EasyConfig to use as the build environment. ' + 'You need to have dependency modules installed already!') + parser.add_argument('--verbose', help='Verbose output', action='store_true') + args = parser.parse_args() - cmd = "source %s/*.env && python %s '%s'" % (tmp_dir, sys.argv[0], args.package) + if args.ec: + if not can_run('eb', '--version'): + print('EasyBuild not found or executable. Make sure it is in your $PATH when using --ec!') + sys.exit(1) if args.verbose: - cmd += ' --verbose' - print('Restarting script in new build environment') - - out = run_cmd(cmd, action_desc='Run in new environment', shell=True, executable='/bin/bash') - print(out) -else: - if not can_run('virtualenv', '--version'): - print('Virtualenv not found or executable. ' + - 'Make sure it is installed (e.g. in the currently loaded Python module)!') - sys.exit(1) - if 'PIP_PREFIX' in os.environ: - print("$PIP_PREFIX is set. Unsetting it as it doesn't work well with virtualenv.") - del os.environ['PIP_PREFIX'] - print_deps(args.package, args.verbose) + print('Checking with EasyBuild for missing dependencies') + missing_dep_out = run_cmd(['eb', args.ec, '--missing'], + capture_stderr=False, + action_desc='Get missing dependencies' + ) + excluded_dep = f'({os.path.basename(args.ec)})' + missing_deps = [dep for dep in missing_dep_out.split('\n') + if dep.startswith('*') and excluded_dep not in dep + ] + if missing_deps: + print(f'You need to install all modules on which {args.ec} depends first!') + print('\n\t'.join(['Missing:'] + missing_deps)) + sys.exit(1) + + # If the --ec argument is a (relative) existing path make it absolute so we can find it after the chdir + ec_arg = os.path.abspath(args.ec) if os.path.exists(args.ec) else args.ec + with temporary_directory() as tmp_dir: + old_dir = os.getcwd() + os.chdir(tmp_dir) + if args.verbose: + print('Running EasyBuild to get build environment') + run_cmd(['eb', ec_arg, '--dump-env', '--force'], action_desc='Dump build environment') + os.chdir(old_dir) + + cmd = f"source {tmp_dir}/*.env && python {sys.argv[0]} '{args.package}'" + if args.verbose: + cmd += ' --verbose' + print('Restarting script in new build environment') + + out = run_cmd(cmd, action_desc='Run in new environment', shell=True, executable='/bin/bash') + print(out) + else: + if not can_run('python', '-m', 'venv', '-h'): + print('Virtualenv not found or executable. ' + + 'Make sure it is installed (e.g. in the currently loaded Python module)!') + sys.exit(1) + if 'PIP_PREFIX' in os.environ: + print("$PIP_PREFIX is set. Unsetting it as it doesn't work well with virtualenv.") + del os.environ['PIP_PREFIX'] + os.environ['PYTHONNOUSERSITE'] = '1' + print_deps(args.package, args.verbose) + + +if __name__ == "__main__": + main() From 36d3a3630b6723f3105183f2bccbff4a30d6c5bb Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 15 Oct 2024 14:10:41 +0200 Subject: [PATCH 6/6] Fix use of venv module --- easybuild/scripts/findPythonDeps.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index 6b57b57ebb..9424cde958 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -96,7 +96,7 @@ def get_dep_tree(package_spec, verbose): venv_dir = os.path.join(tmp_dir, 'venv') if verbose: print('Creating virtualenv at ' + venv_dir) - run_cmd(['virtualenv', '--system-site-packages', venv_dir], action_desc='create virtualenv') + run_cmd([sys.executable, '-m', 'venv', '--system-site-packages', venv_dir], action_desc='create virtualenv') if verbose: print('Updating pip in virtualenv') run_in_venv('pip install --upgrade pip', venv_dir, action_desc='update pip') @@ -241,9 +241,8 @@ def main(): out = run_cmd(cmd, action_desc='Run in new environment', shell=True, executable='/bin/bash') print(out) else: - if not can_run('python', '-m', 'venv', '-h'): - print('Virtualenv not found or executable. ' + - 'Make sure it is installed (e.g. in the currently loaded Python module)!') + if not can_run(sys.executable, '-m', 'venv', '-h'): + print("'venv' module not found. This should be available in Python 3.3+.") sys.exit(1) if 'PIP_PREFIX' in os.environ: print("$PIP_PREFIX is set. Unsetting it as it doesn't work well with virtualenv.")