Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

use run_shell_cmd (and use f-strings) in custom easyblock for GAMESS-US #3342

Merged
merged 2 commits into from
May 30, 2024
Merged

use run_shell_cmd (and use f-strings) in custom easyblock for GAMESS-US #3342

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
08bdc29
replace run_cmd with run_shell_cmd in GAMESS-US easyblock
lexming May 29, 2024
0edfbc9
use f-string in all string formatting in GAMESS-US easyblock
lexming May 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 36 additions & 38 deletions easybuild/easyblocks/g/gamess_us.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
from easybuild.tools.config import build_option
from easybuild.tools.filetools import change_dir, copy_file, mkdir, read_file, write_file, remove_dir
from easybuild.tools.modules import get_software_root, get_software_version
from easybuild.tools.run import run_cmd
from easybuild.tools.run import run_shell_cmd
from easybuild.tools.systemtools import POWER, X86_64
from easybuild.tools.systemtools import get_cpu_architecture
from easybuild.tools import LooseVersion, toolchain
Expand Down Expand Up @@ -142,10 +142,10 @@ def configure_step(self):
fortran_comp, fortran_version = None, None
if comp_fam == toolchain.INTELCOMP:
fortran_comp = 'ifort'
(out, _) = run_cmd("ifort -v", simple=False)
res = re.search(r"^ifort version ([0-9]+)\.[0-9.]+$", out)
res = run_shell_cmd("ifort -v")
regex = re.compile(r"^ifort version ([0-9]+)\.[0-9.]+$")
try:
version_num = res.group(1)
version_num = re.search(regex, res.output).group(1)
except (AttributeError, IndexError):
raise EasyBuildError("Failed to determine ifort major version number")
fortran_version = {"GMS_IFORT_VERNO": version_num}
Expand Down Expand Up @@ -203,7 +203,7 @@ def configure_step(self):

installinfo_opts["GMS_MATHLIB"] = mathlib
installinfo_opts["GMS_MATHLIB_PATH"] = mathlib_path
installinfo_opts["GMS_LAPACK_LINK_LINE"] = '"%s"' % mathlib_flags
installinfo_opts["GMS_LAPACK_LINK_LINE"] = f'"{mathlib_flags}"'

# verify selected DDI communication layer
known_ddi_comms = ['mpi', 'mixed', 'shmem', 'sockets']
Expand Down Expand Up @@ -286,7 +286,7 @@ def configure_step(self):
try:
lked = os.path.join(self.builddir, 'lked')
for line in fileinput.input(lked, inplace=1, backup='.orig'):
line = re.sub(r"^(\s*set\sLIBXC_FLAGS)=.*GMS_PATH.*", r'\1="%s"' % libxc_linker_flags, line)
line = re.sub(r"^(\s*set\sLIBXC_FLAGS)=.*GMS_PATH.*", rf'\1="{libxc_linker_flags}"', line)
sys.stdout.write(line)
except IOError as err:
raise EasyBuildError("Failed to patch %s: %s", lked, err)
Expand Down Expand Up @@ -314,7 +314,7 @@ def configure_step(self):
installinfo_opts['XMVB'] = False

# add include paths from dependencies
installinfo_opts["GMS_FPE_FLAGS"] = '"%s"' % os.environ['CPPFLAGS']
installinfo_opts["GMS_FPE_FLAGS"] = f"\"{os.environ['CPPFLAGS']}\""
# might be useful for debugging
# installinfo_opts["GMS_FPE_FLAGS"] = '"%s"' % os.environ['CPPFLAGS'] + "-ffpe-trap=invalid,zero,overflow"

Expand All @@ -324,27 +324,27 @@ def configure_step(self):
boolean_opts = {opt: str(val).lower() for opt, val in installinfo_opts.items() if val in [True, False]}
installinfo_opts.update(boolean_opts)
# format: setenv KEY VALUE
installinfo_txt = '\n'.join(["setenv %s %s" % (k, installinfo_opts[k]) for k in installinfo_opts])
installinfo_txt = '\n'.join([f"setenv {k} {installinfo_opts[k]}" for k in installinfo_opts])
write_file(installinfo_file, installinfo_txt)
self.log.debug("Contents of %s:\n%s" % (installinfo_file, read_file(installinfo_file)))
self.log.debug(f"Contents of {installinfo_file}:\n{read_file(installinfo_file)}")

# patch hardcoded settings in rungms to use values specified in easyconfig file
rungms = os.path.join(self.builddir, 'rungms')
extra_gmspath_lines = "set ERICFMT=$GMSPATH/auxdata/ericfmt.dat\nset MCPPATH=$GMSPATH/auxdata/MCP\n"
try:
for line in fileinput.input(rungms, inplace=1, backup='.orig'):
line = re.sub(r"^(\s*set\s*TARGET)=.*", r"\1=%s" % self.cfg['ddi_comm'], line)
line = re.sub(r"^(\s*set\s*GMSPATH)=.*", r"\1=%s\n%s" % (self.installdir, extra_gmspath_lines), line)
line = re.sub(r"(null\) set VERNO)=.*", r"\1=%s" % self.version, line)
line = re.sub(r"^(\s*set DDI_MPI_CHOICE)=.*", r"\1=%s" % mpilib, line)
line = re.sub(r"^(\s*set DDI_MPI_ROOT)=.*%s.*" % mpilib.lower(), r"\1=%s" % mpilib_path, line)
line = re.sub(r"^(\s*set GA_MPI_ROOT)=.*%s.*" % mpilib.lower(), r"\1=%s" % mpilib_path, line)
line = re.sub(r"^(\s*set\s*TARGET)=.*", rf"\1={self.cfg['ddi_comm']}", line)
line = re.sub(r"^(\s*set\s*GMSPATH)=.*", rf"\1={self.installdir}\n{extra_gmspath_lines}", line)
line = re.sub(r"(null\) set VERNO)=.*", rf"\1={self.version}", line)
line = re.sub(r"^(\s*set DDI_MPI_CHOICE)=.*", rf"\1={mpilib}", line)
line = re.sub(rf"^(\s*set DDI_MPI_ROOT)=.*{mpilib.lower()}.*", rf"\1={mpilib_path}", line)
line = re.sub(rf"^(\s*set GA_MPI_ROOT)=.*{mpilib.lower()}.*", rf"\1={mpilib_path}", line)
# comment out all adjustments to $LD_LIBRARY_PATH that involves hardcoded paths
line = re.sub(r"^(\s*)(setenv\s*LD_LIBRARY_PATH\s*/.*)", r"\1#\2", line)
# scratch directory paths
line = re.sub(r"^(\s*set\s*SCR)=.*", r"if ( ! $?SCR ) \1=%s" % self.cfg['scratch_dir'], line)
line = re.sub(r"^(\s*set\s*SCR)=.*", rf"if ( ! $?SCR ) \1={self.cfg['scratch_dir']}", line)
line = re.sub(
r"^(\s*set\s*USERSCR)=.*", r"if ( ! $?USERSCR ) \1=%s" % self.cfg['user_scratch_dir'], line
r"^(\s*set\s*USERSCR)=.*", rf"if ( ! $?USERSCR ) \1={self.cfg['user_scratch_dir']}", line
)
line = re.sub(r"^(df -k \$SCR)$", r"mkdir -p $SCR && mkdir -p $USERSCR && \1", line)
if self.cfg['hyperthreading'] is False:
Expand All @@ -359,8 +359,8 @@ def configure_step(self):
compddi = os.path.join(self.builddir, 'ddi/compddi')
try:
for line in fileinput.input(compddi, inplace=1, backup='.orig'):
line = re.sub(r"^(\s*set MAXCPUS)=.*", r"\1=%s" % self.cfg['maxcpus'], line, 1)
line = re.sub(r"^(\s*set MAXNODES)=.*", r"\1=%s" % self.cfg['maxnodes'], line, 1)
line = re.sub(r"^(\s*set MAXCPUS)=.*", rf"\1={self.cfg['maxcpus']}", line, 1)
line = re.sub(r"^(\s*set MAXNODES)=.*", rf"\1={self.cfg['maxnodes']}", line, 1)
sys.stdout.write(line)
except IOError as err:
raise EasyBuildError("Failed to patch compddi", compddi, err)
Expand All @@ -375,39 +375,37 @@ def configure_step(self):
except IOError as err:
raise EasyBuildError("Failed to patch actvte.code", actvte, err)
# compiling
run_cmd("mv %s/tools/actvte.code" % self.builddir + " %s/tools/actvte.f" % self.builddir)
run_cmd(
"%s -o " % fortran_comp + " %s/tools/actvte.x" % self.builddir + " %s/tools/actvte.f" % self.builddir
)
run_shell_cmd(f"mv {self.builddir}/tools/actvte.code {self.builddir}/tools/actvte.f")
run_shell_cmd(f"{fortran_comp} -o {self.builddir}/tools/actvte.x {self.builddir}/tools/actvte.f")

def build_step(self):
"""Custom build procedure for GAMESS-US: using compddi, compall and lked scripts."""
compddi = os.path.join(self.cfg['start_dir'], 'ddi', 'compddi')
run_cmd(compddi, log_all=True, simple=True)
run_shell_cmd(compddi)

# make sure the libddi.a library is present
libddi = os.path.join(self.cfg['start_dir'], 'ddi', 'libddi.a')
if not os.path.isfile(libddi):
raise EasyBuildError("The libddi.a library (%s) was never built", libddi)
else:
self.log.info("The libddi.a library (%s) was successfully built." % libddi)
self.log.info("The libddi.a library (%s) was successfully built.", libddi)

ddikick = os.path.join(self.cfg['start_dir'], 'ddi', 'ddikick.x')
if os.path.isfile(ddikick):
self.log.info("The ddikick.x executable (%s) was successfully built." % ddikick)
self.log.info("The ddikick.x executable (%s) was successfully built.", ddikick)

if self.cfg['ddi_comm'] == 'sockets':
src = ddikick
dst = os.path.join(self.cfg['start_dir'], 'ddikick.x')
self.log.info("Moving ddikick.x executable from %s to %s." % (src, dst))
self.log.info("Moving ddikick.x executable from %s to %s.", src, dst)
os.rename(src, dst)

compall_cmd = os.path.join(self.cfg['start_dir'], 'compall')
compall = "%s %s %s" % (self.cfg['prebuildopts'], compall_cmd, self.cfg['buildopts'])
run_cmd(compall, log_all=True, simple=True)
compall = " ".join([self.cfg['prebuildopts'], compall_cmd, self.cfg['buildopts']])
run_shell_cmd(compall)

cmd = "%s gamess %s" % (os.path.join(self.cfg['start_dir'], 'lked'), self.version)
run_cmd(cmd, log_all=True, simple=True)
gamess_cmd = f"{os.path.join(self.cfg['start_dir'], 'lked')} gamess {self.version}"
run_shell_cmd(gamess_cmd)

def test_step(self):
"""Run GAMESS-US tests (if 'runtest' easyconfig parameter is set to True)."""
Expand Down Expand Up @@ -457,27 +455,27 @@ def test_step(self):
except OSError as err:
raise EasyBuildError("Failed to copy test '%s' to %s: %s", exam, self.testdir, err)

test_env_vars.append('SCR=%s' % self.testdir)
test_env_vars.append(f"SCR={self.testdir}")

# run target exam<id> tests, dump output to exam<id>.log
rungms = os.path.join(self.installdir, 'rungms')
for exam, exam_file in target_tests:
rungms_prefix = ' && '.join(test_env_vars)
test_cmd = [rungms_prefix, rungms, exam_file, self.version, test_procs, test_procs]
(out, _) = run_cmd(' '.join(test_cmd), log_all=True, simple=False)
write_file('%s.log' % exam, out)
res = run_shell_cmd(' '.join(test_cmd))
write_file(f"{exam}.log", res.output)

check_cmd = os.path.join(self.installdir, 'tests', 'standard', 'checktst')
(out, _) = run_cmd(check_cmd, log_all=True, simple=False)
res = run_shell_cmd(check_cmd)

# verify output of tests
failed_regex = re.compile(r"^.*!!FAILED\.$", re.M)
failed_tests = set([exam[0:6] for exam in failed_regex.findall(out)])
failed_tests = set([exam[0:6] for exam in failed_regex.findall(res.output)])
done_tests = set([exam[0] for exam in target_tests])
if done_tests - failed_tests == done_tests:
info_msg = "All target tests ran successfully!"
if self.cfg['ddi_comm'] == 'mpi':
info_msg += " (serial tests ignored: %s)" % ", ".join(GAMESS_SERIAL_TESTS)
info_msg += f" (serial tests ignored: {', '.join(GAMESS_SERIAL_TESTS)})"
self.log.info(info_msg)
else:
raise EasyBuildError("ERROR: Not all target tests ran successfully")
Expand All @@ -496,7 +494,7 @@ def install_step(self):
def sanity_check_step(self):
"""Custom sanity check for GAMESS-US."""
custom_paths = {
'files': ['gamess.%s.x' % self.version, 'rungms'],
'files': [f'gamess.{self.version}.x', 'rungms'],
'dirs': [],
}
super(EB_GAMESS_minus_US, self).sanity_check_step(custom_paths=custom_paths)
Expand Down
Loading