Skip to content

Commit

Permalink
Merge branch '5.0.x' into run_shell_cmd_qa
Browse files Browse the repository at this point in the history
  • Loading branch information
boegel committed Mar 2, 2024
2 parents 4b63b91 + 849ffed commit 23fa25c
Show file tree
Hide file tree
Showing 15 changed files with 106 additions and 36 deletions.
14 changes: 11 additions & 3 deletions .github/workflows/container_tests_apptainer.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,18 @@ jobs:
- name: install OS & Python packages
run: |
# for building CentOS 7 container images
sudo apt-get install rpm
sudo apt-get install dnf
APT_PKGS="rpm dnf"
# for modules tool
sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
APT_PKGS+=" lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev"
# Avoid apt-get update, as we don't really need it,
# and it does more harm than good (it's fairly expensive, and it results in flaky test runs)
if ! sudo apt-get install $APT_PKGS; then
# Try to update cache, then try again to resolve 404s of old packages
sudo apt-get update -yqq || true
sudo apt-get install $APT_PKGS
fi
# fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
# needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/unit_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ jobs:
echo "Not testing with '${module_syntax}' as module syntax with '${EASYBUILD_MODULES_TOOL}' as modules tool"
continue
fi
printf '\n\n=====================> Using $module_syntax module syntax <=====================\n\n'
printf "\n\n=====================> Using $module_syntax module syntax <=====================\n\n"
export EASYBUILD_MODULE_SYNTAX="${module_syntax}"
export TEST_EASYBUILD_MODULE_SYNTAX="${EASYBUILD_MODULE_SYNTAX}"
Expand Down
4 changes: 3 additions & 1 deletion easybuild/framework/easyblock.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
import traceback
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
from textwrap import indent

import easybuild.tools.environment as env
import easybuild.tools.toolchain as toolchain
Expand Down Expand Up @@ -953,7 +954,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
if download_instructions is None:
download_instructions = self.cfg['download_instructions']
if download_instructions is not None and download_instructions != "":
msg = "\nDownload instructions:\n\n" + download_instructions + '\n'
msg = "\nDownload instructions:\n\n" + indent(download_instructions, ' ') + '\n\n'
msg += "Make the files available in the active source path: %s\n" % ':'.join(source_paths())
print_msg(msg, prefix=False, stderr=True)
error_msg += "please follow the download instructions above, and make the file available "
error_msg += "in the active source path (%s)" % ':'.join(source_paths())
Expand Down
7 changes: 7 additions & 0 deletions easybuild/framework/easyconfig/templates.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,9 @@
('cuda_cc_cmake', "List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+"),
('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"),
('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"),
('cuda_int_comma_sep', "Comma-separated list of integer CUDA compute capabilities"),
('cuda_int_space_sep', "Space-separated list of integer CUDA compute capabilities"),
('cuda_int_semicolon_sep', "Semicolon-separated list of integer CUDA compute capabilities"),
('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"),
('cuda_sm_space_sep', "Space-separated list of sm_* values that correspond with CUDA compute capabilities"),
]
Expand Down Expand Up @@ -365,6 +368,10 @@ def template_constant_dict(config, ignore=None, toolchain=None):
template_values['cuda_cc_space_sep'] = ' '.join(cuda_compute_capabilities)
template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_compute_capabilities)
template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_compute_capabilities)
int_values = [cc.replace('.', '') for cc in cuda_compute_capabilities]
template_values['cuda_int_comma_sep'] = ','.join(int_values)
template_values['cuda_int_space_sep'] = ' '.join(int_values)
template_values['cuda_int_semicolon_sep'] = ';'.join(int_values)
sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_compute_capabilities]
template_values['cuda_sm_comma_sep'] = ','.join(sm_values)
template_values['cuda_sm_space_sep'] = ' '.join(sm_values)
Expand Down
2 changes: 1 addition & 1 deletion easybuild/tools/filetools.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@
# tar.Z: using compress (LZW), but can be handled with gzip so use 'z'
'.tar.z': "tar xzf %(filepath)s",
# shell scripts don't need to be unpacked, just copy there
'.sh': "cp -a %(filepath)s .",
'.sh': "cp -dR %(filepath)s .",
}

ZIPPED_PATCH_EXTS = ('.bz2', '.gz', '.xz')
Expand Down
3 changes: 1 addition & 2 deletions easybuild/tools/github.py
Original file line number Diff line number Diff line change
Expand Up @@ -1371,8 +1371,7 @@ def close_pr(pr, motivation_msg=None):
if not reopen:
msg += "\nPlease don't hesitate to reopen this PR or add a comment if you feel this contribution is still "
msg += "relevant.\nFor more information on our policy w.r.t. closing PRs, see "
msg += "https://easybuild.readthedocs.io/en/latest/Contributing.html"
msg += "#why-a-pull-request-may-be-closed-by-a-maintainer"
msg += "https://docs.easybuild.io/contributing/#contributing_review_process_why_pr_closed_by_maintainer"
post_comment_in_issue(pr, msg, account=pr_target_account, repo=pr_target_repo, github_user=github_user)

if dry_run:
Expand Down
3 changes: 2 additions & 1 deletion easybuild/tools/options.py
Original file line number Diff line number Diff line change
Expand Up @@ -353,7 +353,8 @@ def override_options(self):
None, 'store_true', False),
'allow-use-as-root-and-accept-consequences': ("Allow using of EasyBuild as root (NOT RECOMMENDED!)",
None, 'store_true', False),
'backup-modules': ("Back up an existing module file, if any. Only works when using --module-only",
'backup-modules': ("Back up an existing module file, if any. "
"Auto-enabled when using --module-only or --skip",
None, 'store_true', None), # default None to allow auto-enabling if not disabled
'backup-patched-files': ("Create a backup (*.orig) file when applying a patch",
None, 'store_true', False),
Expand Down
16 changes: 15 additions & 1 deletion easybuild/tools/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import re
import signal
import shutil
import string
import subprocess
import sys
import tempfile
Expand Down Expand Up @@ -188,6 +189,19 @@ def cache_aware_func(cmd, *args, **kwargs):
run_shell_cmd_cache = run_cmd_cache


def fileprefix_from_cmd(cmd, allowed_chars=False):
"""
Simplify the cmd to only the allowed_chars we want in a filename
:param cmd: the cmd (string)
:param allowed_chars: characters allowed in filename (defaults to string.ascii_letters + string.digits + "_-")
"""
if not allowed_chars:
allowed_chars = f"{string.ascii_letters}{string.digits}_-"

return ''.join([c for c in cmd if c in allowed_chars])


@run_shell_cmd_cache
def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None,
hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, use_bash=True,
Expand Down Expand Up @@ -246,7 +260,6 @@ def to_cmd_str(cmd):
work_dir = os.getcwd()

cmd_str = to_cmd_str(cmd)
cmd_name = os.path.basename(cmd_str.split(' ')[0])

thread_id = None
if asynchronous:
Expand All @@ -262,6 +275,7 @@ def to_cmd_str(cmd):
if output_file:
toptmpdir = os.path.join(tempfile.gettempdir(), 'run-shell-cmd-output')
os.makedirs(toptmpdir, exist_ok=True)
cmd_name = fileprefix_from_cmd(os.path.basename(cmd_str.split(' ')[0]))
tmpdir = tempfile.mkdtemp(dir=toptmpdir, prefix=f'{cmd_name}-')
cmd_out_fp = os.path.join(tmpdir, 'out.txt')
_log.info(f'run_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}')
Expand Down
12 changes: 8 additions & 4 deletions test/framework/easyblock.py
Original file line number Diff line number Diff line change
Expand Up @@ -1582,7 +1582,8 @@ def test_download_instructions(self):
self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step)
stderr = self.get_stderr().strip()
self.mock_stderr(False)
self.assertIn("Download instructions:\n\nManual download from example.com required", stderr)
self.assertIn("Download instructions:\n\n Manual download from example.com required", stderr)
self.assertIn("Make the files available in the active source path", stderr)

# create dummy source file
write_file(os.path.join(os.path.dirname(self.eb_file), 'software_with_missing_sources-0.0.tar.gz'), '')
Expand All @@ -1596,7 +1597,8 @@ def test_download_instructions(self):
stderr = self.get_stderr().strip()
self.mock_stderr(False)
self.mock_stdout(False)
self.assertIn("Download instructions:\n\nManual download from example.com required", stderr)
self.assertIn("Download instructions:\n\n Manual download from example.com required", stderr)
self.assertIn("Make the files available in the active source path", stderr)

# wipe top-level download instructions, try again
self.contents = self.contents.replace(download_instructions, '')
Expand Down Expand Up @@ -1625,7 +1627,8 @@ def test_download_instructions(self):
stderr = self.get_stderr().strip()
self.mock_stderr(False)
self.mock_stdout(False)
self.assertIn("Download instructions:\n\nExtension sources must be downloaded via example.com", stderr)
self.assertIn("Download instructions:\n\n Extension sources must be downloaded via example.com", stderr)
self.assertIn("Make the files available in the active source path", stderr)

# download instructions should also be printed if 'source_tmpl' is used to specify extension sources
self.contents = self.contents.replace(sources, "'source_tmpl': SOURCE_TAR_GZ,")
Expand All @@ -1638,7 +1641,8 @@ def test_download_instructions(self):
stderr = self.get_stderr().strip()
self.mock_stderr(False)
self.mock_stdout(False)
self.assertIn("Download instructions:\n\nExtension sources must be downloaded via example.com", stderr)
self.assertIn("Download instructions:\n\n Extension sources must be downloaded via example.com", stderr)
self.assertIn("Make the files available in the active source path", stderr)

# create dummy source file for extension
write_file(os.path.join(os.path.dirname(self.eb_file), 'ext_with_missing_sources-0.0.tar.gz'), '')
Expand Down
35 changes: 23 additions & 12 deletions test/framework/easyconfig.py
Original file line number Diff line number Diff line change
Expand Up @@ -4438,31 +4438,39 @@ def test_cuda_compute_capabilities(self):
description = 'test'
toolchain = SYSTEM
cuda_compute_capabilities = ['5.1', '7.0', '7.1']
installopts = '%(cuda_compute_capabilities)s'
preinstallopts = '%(cuda_cc_space_sep)s'
prebuildopts = '%(cuda_cc_semicolon_sep)s'
configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"'
preconfigopts = 'CUDAARCHS="%(cuda_cc_cmake)s"'
configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"'
prebuildopts = '%(cuda_cc_semicolon_sep)s'
buildopts = ('comma="%(cuda_int_comma_sep)s" space="%(cuda_int_space_sep)s" '
'semi="%(cuda_int_semicolon_sep)s"')
preinstallopts = '%(cuda_cc_space_sep)s'
installopts = '%(cuda_compute_capabilities)s'
""")
self.prep()

ec = EasyConfig(self.eb_file)
self.assertEqual(ec['installopts'], '5.1,7.0,7.1')
self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1')
self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1')
self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="51;70;71"')
self.assertEqual(ec['configopts'], 'comma="sm_51,sm_70,sm_71" '
'space="sm_51 sm_70 sm_71"')
self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="51;70;71"')
self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1')
self.assertEqual(ec['buildopts'], 'comma="51,70,71" '
'space="51 70 71" '
'semi="51;70;71"')
self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1')
self.assertEqual(ec['installopts'], '5.1,7.0,7.1')

# build options overwrite it
init_config(build_options={'cuda_compute_capabilities': ['4.2', '6.3']})
ec = EasyConfig(self.eb_file)
self.assertEqual(ec['installopts'], '4.2,6.3')
self.assertEqual(ec['preinstallopts'], '4.2 6.3')
self.assertEqual(ec['prebuildopts'], '4.2;6.3')
self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="42;63"')
self.assertEqual(ec['configopts'], 'comma="sm_42,sm_63" '
'space="sm_42 sm_63"')
self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="42;63"')
self.assertEqual(ec['buildopts'], 'comma="42,63" '
'space="42 63" '
'semi="42;63"')
self.assertEqual(ec['prebuildopts'], '4.2;6.3')
self.assertEqual(ec['preinstallopts'], '4.2 6.3')
self.assertEqual(ec['installopts'], '4.2,6.3')

def test_det_copy_ec_specs(self):
"""Test det_copy_ec_specs function."""
Expand Down Expand Up @@ -4725,6 +4733,9 @@ def test_get_cuda_cc_template_value(self):
'cuda_compute_capabilities': '6.5,7.0',
'cuda_cc_space_sep': '6.5 7.0',
'cuda_cc_semicolon_sep': '6.5;7.0',
'cuda_int_comma_sep': '65,70',
'cuda_int_space_sep': '65 70',
'cuda_int_semicolon_sep': '65;70',
'cuda_sm_comma_sep': 'sm_65,sm_70',
'cuda_sm_space_sep': 'sm_65 sm_70',
}
Expand Down
2 changes: 1 addition & 1 deletion test/framework/filetools.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ def test_extract_cmd(self):
('test.txz', "unset TAPE; unxz test.txz --stdout | tar x"),
('test.iso', "7z x test.iso"),
('test.tar.Z', "tar xzf test.tar.Z"),
('test.foo.bar.sh', "cp -a test.foo.bar.sh ."),
('test.foo.bar.sh', "cp -dR test.foo.bar.sh ."),
# check whether extension is stripped correct to determine name of target file
# cfr. https://github.com/easybuilders/easybuild-framework/pull/3705
('testbz2.bz2', "bunzip2 -c testbz2.bz2 > testbz2"),
Expand Down
15 changes: 9 additions & 6 deletions test/framework/options.py
Original file line number Diff line number Diff line change
Expand Up @@ -4905,26 +4905,29 @@ def test_github_merge_pr(self):
self.assertEqual(stderr.strip(), expected_stderr)
self.assertTrue(stdout.strip().endswith(expected_stdout), "'%s' ends with '%s'" % (stdout, expected_stdout))

# full eligible merged PR, default target branch
# full eligible merged PR, default target branch;
# note: we frequently need to change to a more recent PR here,
# to avoid that this test starts failing because commit status is set to None for old commits
del args[-1]
args[1] = '17065'
# easyconfig PR for EasyBuild v4.8.2
args[1] = '19105'

stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False)

expected_stdout = '\n'.join([
"Checking eligibility of easybuilders/easybuild-easyconfigs PR #17065 for merging...",
"Checking eligibility of easybuilders/easybuild-easyconfigs PR #19105 for merging...",
"* targets develop branch: OK",
"* test suite passes: OK",
"* last test report is successful: OK",
"* no pending change requests: OK",
"* approved review: OK (by SebastianAchilles)",
"* milestone is set: OK (4.7.1)",
"* milestone is set: OK (4.9.0)",
"* mergeable state is clean: PR is already merged",
'',
"Review OK, merging pull request!",
'',
"[DRY RUN] Adding comment to easybuild-easyconfigs issue #17065: 'Going in, thanks @boegel!'",
"[DRY RUN] Merged easybuilders/easybuild-easyconfigs pull request #17065",
"[DRY RUN] Adding comment to easybuild-easyconfigs issue #19105: 'Going in, thanks @boegel!'",
"[DRY RUN] Merged easybuilders/easybuild-easyconfigs pull request #19105",
])
expected_stderr = ''
self.assertEqual(stderr.strip(), expected_stderr)
Expand Down
23 changes: 22 additions & 1 deletion test/framework/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import os
import re
import signal
import string
import stat
import subprocess
import sys
Expand All @@ -52,7 +53,7 @@
from easybuild.tools.config import update_build_option
from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir, read_file, write_file
from easybuild.tools.run import RunShellCmdResult, RunShellCmdError, check_async_cmd, check_log_for_errors
from easybuild.tools.run import complete_cmd, get_output_from_process, parse_log_for_error
from easybuild.tools.run import complete_cmd, fileprefix_from_cmd, get_output_from_process, parse_log_for_error
from easybuild.tools.run import run_cmd, run_cmd_qa, run_shell_cmd, subprocess_terminate
from easybuild.tools.config import ERROR, IGNORE, WARN

Expand Down Expand Up @@ -194,6 +195,26 @@ def test_run_shell_cmd_basic(self):
self.assertTrue(isinstance(res.output, str))
self.assertTrue(res.work_dir and isinstance(res.work_dir, str))

def test_fileprefix_from_cmd(self):
"""test simplifications from fileprefix_from_cmd."""
cmds = {
'abd123': 'abd123',
'ab"a': 'aba',
'a{:$:S@"a': 'aSa',
'cmd-with-dash': 'cmd-with-dash',
'cmd_with_underscore': 'cmd_with_underscore',
}
for cmd, expected_simplification in cmds.items():
self.assertEqual(fileprefix_from_cmd(cmd), expected_simplification)

cmds = {
'abd123': 'abd',
'ab"a': 'aba',
'0a{:$:2@"a': 'aa',
}
for cmd, expected_simplification in cmds.items():
self.assertEqual(fileprefix_from_cmd(cmd, allowed_chars=string.ascii_letters), expected_simplification)

def test_run_cmd_log(self):
"""Test logging of executed commands."""
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ def run_async(self, thread_pool):
cmd = f"echo 'no sources for {self.name}'"

return thread_pool.submit(run_shell_cmd, cmd, asynchronous=True, env=os.environ.copy(),
fail_on_error=False, task_id=task_id)
fail_on_error=False, task_id=task_id, work_dir=os.getcwd())

def postrun(self):
"""
Expand Down
2 changes: 1 addition & 1 deletion test/framework/sandbox/easybuild/easyblocks/t/toy.py
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ def run_async(self, thread_pool):
cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts'])
task_id = f'ext_{self.name}_{self.version}'
return thread_pool.submit(run_shell_cmd, cmd, asynchronous=True, env=os.environ.copy(),
fail_on_error=False, task_id=task_id)
fail_on_error=False, task_id=task_id, work_dir=os.getcwd())

def postrun(self):
"""
Expand Down

0 comments on commit 23fa25c

Please sign in to comment.