Skip to content

Commit

Permalink
Merge pull request #3319 from akesandgren/fix-crayce-problems
Browse files Browse the repository at this point in the history
Fix problems with CrayCCE processing when there are no actual external modules
  • Loading branch information
boegel authored May 3, 2020
2 parents ed5ba4a + 1a58c28 commit 0f78649
Show file tree
Hide file tree
Showing 7 changed files with 156 additions and 48 deletions.
2 changes: 1 addition & 1 deletion easybuild/toolchains/mpi/craympich.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ class CrayMPICH(Mpi):
"""Generic support for using Cray compiler wrappers"""
# MPI support
# no separate module, Cray compiler drivers always provide MPI support
MPI_MODULE_NAME = []
MPI_MODULE_NAME = None
MPI_FAMILY = TC_CONSTANT_MPICH
MPI_TYPE = TC_CONSTANT_MPI_TYPE_MPICH

Expand Down
16 changes: 15 additions & 1 deletion easybuild/tools/module_naming_scheme/hierarchical_mns.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,11 @@
CORE = 'Core'
COMPILER = 'Compiler'
MPI = 'MPI'
TOOLCHAIN = 'Toolchain'

MODULECLASS_COMPILER = 'compiler'
MODULECLASS_MPI = 'mpi'
MODULECLASS_TOOLCHAIN = 'toolchain'

GCCCORE = GCCcore.NAME

Expand Down Expand Up @@ -107,7 +109,11 @@ def det_toolchain_compilers_name_version(self, tc_comps):
# no compiler in toolchain, system toolchain
res = None
elif len(tc_comps) == 1:
res = (tc_comps[0]['name'], self.det_full_version(tc_comps[0]))
tc_comp = tc_comps[0]
if tc_comp is None:
res = None
else:
res = (tc_comp['name'], self.det_full_version(tc_comp))
else:
comp_versions = dict([(comp['name'], self.det_full_version(comp)) for comp in tc_comps])
comp_names = comp_versions.keys()
Expand Down Expand Up @@ -135,6 +141,10 @@ def det_module_subdir(self, ec):
if tc_comps is None:
# no compiler in toolchain, system toolchain => Core module
subdir = CORE
elif tc_comps == [None]:
# no info on toolchain compiler (cfr. Cray toolchains),
# then use toolchain name/version
subdir = os.path.join(TOOLCHAIN, ec.toolchain.name, ec.toolchain.version)
else:
tc_comp_name, tc_comp_ver = self.det_toolchain_compilers_name_version(tc_comps)
tc_mpi = det_toolchain_mpi(ec)
Expand Down Expand Up @@ -223,6 +233,10 @@ def det_modpath_extensions(self, ec):
fullver = self.det_full_version(ec)
paths.append(os.path.join(MPI, tc_comp_name, tc_comp_ver, ec['name'], fullver))

# special case for Cray toolchains
elif modclass == MODULECLASS_TOOLCHAIN and tc_comp_info is None and ec.name.startswith('Cray'):
paths.append(os.path.join(TOOLCHAIN, ec.name, ec.version))

return paths

def expand_toolchain_load(self, ec=None):
Expand Down
3 changes: 2 additions & 1 deletion easybuild/tools/toolchain/mpi.py
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,8 @@ def _set_mpi_variables(self):
if not self.options.get('32bit', None):
suffix = '64'

for root in self.get_software_root(self.MPI_MODULE_NAME):
# take into account that MPI_MODULE_NAME could be None (see Cray toolchains)
for root in self.get_software_root(self.MPI_MODULE_NAME or []):
self.variables.append_exists('MPI_LIB_STATIC', root, lib_dir, filename="lib%s.a" % self.MPI_LIBRARY_NAME,
suffix=suffix)
self.variables.append_exists('MPI_LIB_SHARED', root, lib_dir, filename="lib%s.so" % self.MPI_LIBRARY_NAME,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
easyblock = 'ConfigureMake'

name = 'HPL'
version = '2.1'

homepage = 'http://www.netlib.org/benchmark/hpl/'
description = "HPL, you know, LINPACK"

toolchain = {'name': 'CrayCCE', 'version': '5.1.29'}

source_urls = ['http://www.netlib.org/benchmark/%(namelower)s']
sources = [SOURCELOWER_TAR_GZ]

moduleclass = 'tools'
2 changes: 1 addition & 1 deletion test/framework/filetools.py
Original file line number Diff line number Diff line change
Expand Up @@ -1816,7 +1816,7 @@ def test_index_functions(self):
# test with specified path with and without trailing '/'s
for path in [test_ecs, test_ecs + '/', test_ecs + '//']:
index = ft.create_index(path)
self.assertEqual(len(index), 81)
self.assertEqual(len(index), 82)

expected = [
os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'),
Expand Down
5 changes: 5 additions & 0 deletions test/framework/module_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -1273,6 +1273,11 @@ def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts,
['MPI/intel-CUDA/%s-5.5.22/impi/5.1.2.150' % iccver],
['MPI/intel-CUDA/%s-5.5.22/impi/5.1.2.150' % iccver],
['Core']),
'CrayCCE-5.1.29.eb': ('CrayCCE/5.1.29', 'Core',
['Toolchain/CrayCCE/5.1.29'],
['Toolchain/CrayCCE/5.1.29'],
['Core']),
'HPL-2.1-CrayCCE-5.1.29.eb': ('HPL/2.1', 'Toolchain/CrayCCE/5.1.29', [], [], ['Core']),
}
for ecfile, mns_vals in test_ecs.items():
test_ec(ecfile, *mns_vals)
Expand Down
162 changes: 118 additions & 44 deletions test/framework/options.py
Original file line number Diff line number Diff line change
Expand Up @@ -4265,54 +4265,128 @@ def test_list_prs(self):

def test_list_software(self):
"""Test --list-software and --list-installed-software."""
test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'v1.0')
args = [
'--list-software',
'--robot-paths=%s' % test_ecs,
]
txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False)
expected = '\n'.join([
"== Processed 5/5 easyconfigs...",
"== Found 2 different software packages",
'',
"* GCC",
"* gzip",
'',

# copy selected test easyconfigs for testing --list-*software options with;
# full test is a nuisance, because all dependencies must be available and toolchains like intel must have
# all expected components when testing with HierarchicalMNS (which the test easyconfigs don't always have)
topdir = os.path.dirname(os.path.abspath(__file__))

cray_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'c', 'CrayCCE', 'CrayCCE-5.1.29.eb')
gcc_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'GCC', 'GCC-4.6.3.eb')
gzip_ec = os.path.join(topdir, 'easyconfigs', 'v1.0', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb')
gzip_system_ec = os.path.join(topdir, 'easyconfigs', 'v1.0', 'g', 'gzip', 'gzip-1.4.eb')

test_ecs = os.path.join(self.test_prefix, 'test_ecs')
for ec in [cray_ec, gcc_ec, gzip_ec, gzip_system_ec]:
subdirs = os.path.dirname(ec).split(os.path.sep)[-2:]
target_dir = os.path.join(test_ecs, *subdirs)
mkdir(target_dir, parents=True)
copy_file(ec, target_dir)

# add (fake) HPL easyconfig using CrayCCE toolchain
# (required to trigger bug reported in https://github.com/easybuilders/easybuild-framework/issues/3265)
hpl_cray_ec_txt = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "HPL"',
'version = "2.3"',
"homepage = 'http://www.netlib.org/benchmark/hpl/'",
'description = "HPL"',
'toolchain = {"name": "CrayCCE", "version": "5.1.29"}',
])
self.assertTrue(txt.endswith(expected))
hpl_cray_ec = os.path.join(self.test_prefix, 'test_ecs', 'h', 'HPL', 'HPL-2.3-CrayCCE-5.1.29.eb')
write_file(hpl_cray_ec, hpl_cray_ec_txt)

args = [
'--list-software=detailed',
'--output-format=rst',
'--robot-paths=%s' % test_ecs,
]
txt, _ = self._run_mock_eb(args, testing=False)
self.assertTrue(re.search(r'^\*GCC\*', txt, re.M))
self.assertTrue(re.search(r'^``4.6.3``\s+``system``', txt, re.M))
self.assertTrue(re.search(r'^\*gzip\*', txt, re.M))
self.assertTrue(re.search(r'^``1.5``\s+``foss/2018a``,\s+``intel/2018a``', txt, re.M))
# put dummy Core/GCC/4.6.3 in place
modpath = os.path.join(self.test_prefix, 'modules')
write_file(os.path.join(modpath, 'Core', 'GCC', '4.6.3'), '#%Module')
self.modtool.use(modpath)

args = [
'--list-installed-software',
'--output-format=rst',
'--robot-paths=%s' % test_ecs,
]
txt, _ = self._run_mock_eb(args, testing=False, raise_error=True)
self.assertTrue(re.search(r'== Processed 5/5 easyconfigs...', txt, re.M))
self.assertTrue(re.search(r'== Found 2 different software packages', txt, re.M))
self.assertTrue(re.search(r'== Retained 1 installed software packages', txt, re.M))
self.assertTrue(re.search(r'^\* GCC', txt, re.M))
self.assertFalse(re.search(r'gzip', txt, re.M))
# test with different module naming scheme active
# (see https://github.com/easybuilders/easybuild-framework/issues/3265)
for mns in ['EasyBuildMNS', 'HierarchicalMNS']:

args = [
'--list-installed-software=detailed',
'--robot-paths=%s' % test_ecs,
]
txt, _ = self._run_mock_eb(args, testing=False)
self.assertTrue(re.search(r'^== Retained 1 installed software packages', txt, re.M))
self.assertTrue(re.search(r'^\* GCC', txt, re.M))
self.assertTrue(re.search(r'^\s+\* GCC v4.6.3: system', txt, re.M))
self.assertFalse(re.search(r'gzip', txt, re.M))
args = [
'--list-software',
'--robot-paths=%s' % test_ecs,
'--module-naming-scheme=%s' % mns,
]
txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, verbose=True)

patterns = [
r"^.*\s*== Processed 5/5 easyconfigs...",
r"^== Found 4 different software packages",
r"^\* CrayCCE",
r"^\* GCC",
r"^\* gzip",
r"^\* HPL",
]
for pattern in patterns:
regex = re.compile(pattern, re.M)
self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))

args = [
'--list-software=detailed',
'--output-format=rst',
'--robot-paths=%s' % test_ecs,
'--module-naming-scheme=%s' % mns,
]
txt, _ = self._run_mock_eb(args, testing=False, raise_error=True, verbose=True)

patterns = [
r"^.*\s*== Processed 5/5 easyconfigs...",
r"^== Found 4 different software packages",
r'^\*CrayCCE\*',
r'^``5.1.29``\s+``system``',
r'^\*GCC\*',
r'^``4.6.3``\s+``system``',
r'^\*gzip\*',
r'^``1.4`` ``GCC/4.6.3``, ``system``',
]
for pattern in patterns:
regex = re.compile(pattern, re.M)
self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))

args = [
'--list-installed-software',
'--output-format=rst',
'--robot-paths=%s' % test_ecs,
'--module-naming-scheme=%s' % mns,
]
txt, _ = self._run_mock_eb(args, testing=False, raise_error=True, verbose=True)

patterns = [
r"^.*\s*== Processed 5/5 easyconfigs...",
r"^== Found 4 different software packages",
r"^== Retained 1 installed software packages",
r'^\* GCC',
]
for pattern in patterns:
regex = re.compile(pattern, re.M)
self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))

self.assertFalse(re.search(r'gzip', txt, re.M))
self.assertFalse(re.search(r'CrayCCE', txt, re.M))

args = [
'--list-installed-software=detailed',
'--robot-paths=%s' % test_ecs,
'--module-naming-scheme=%s' % mns,
]
txt, _ = self._run_mock_eb(args, testing=False, raise_error=True, verbose=True)

patterns = [
r"^.*\s*== Processed 5/5 easyconfigs...",
r"^== Found 4 different software packages",
r"^== Retained 1 installed software packages",
r'^\* GCC',
r'^\s+\* GCC v4.6.3: system',
]
for pattern in patterns:
regex = re.compile(pattern, re.M)
self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))

self.assertFalse(re.search(r'gzip', txt, re.M))
self.assertFalse(re.search(r'CrayCCE', txt, re.M))

def test_parse_optarch(self):
"""Test correct parsing of optarch option."""
Expand Down

0 comments on commit 0f78649

Please sign in to comment.