Skip to content

Commit

Permalink
Allow passing arch-specific optarch on cmdline/config
Browse files Browse the repository at this point in the history
  • Loading branch information
@Flamefire
Flamefire committed Aug 11, 2021
1 parent ec51396 commit 6298aa9
Show file tree
Hide file tree
Showing 5 changed files with 114 additions and 41 deletions.
78 changes: 58 additions & 20 deletions easybuild/tools/options.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,8 @@
from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, OPTARCH_MAP_CHAR, OPTARCH_SEP, Compiler
from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME
from easybuild.tools.repository.repository import avail_repositories
from easybuild.tools.systemtools import UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family
from easybuild.tools.systemtools import UNKNOWN, CPU_ARCHITECTURES, CPU_FAMILIES, CPU_VECTOR_EXTS
from easybuild.tools.systemtools import check_python_version, get_cpu_architecture, get_cpu_family
from easybuild.tools.systemtools import get_cpu_features, get_system_info
from easybuild.tools.version import this_is_easybuild

Expand Down Expand Up @@ -930,29 +931,66 @@ def postprocess(self):
cleanup_and_exit(self.tmpdir)

def _postprocess_optarch(self):
"""Postprocess --optarch option."""
optarch_parts = self.options.optarch.split(OPTARCH_SEP)

# we expect to find a ':' in every entry in optarch, in case optarch is specified on a per-compiler basis
n_parts = len(optarch_parts)
map_char_cnts = [p.count(OPTARCH_MAP_CHAR) for p in optarch_parts]
if (n_parts > 1 and any(c != 1 for c in map_char_cnts)) or (n_parts == 1 and map_char_cnts[0] > 1):
"""
Postprocess --optarch option.
Format: map := <entry>(;<space>*<entry>)*
entry := <compiler>(,<arch-spec)*:<flag-string>
Example values:
"GENERIC"
"march=native"
"Intel,x86:xHost; Intel,x86,AMD,AVX2:mavx2 -fma; GCC:march=native"
"""
# Split into entries, and each entry into key-value pairs
optarch_parts = [i.strip().split(OPTARCH_MAP_CHAR) for i in self.options.optarch.split(OPTARCH_SEP)]

# We should either have a single value or a list of key-value pairs, and nothing else
is_single_value = len(optarch_parts) == 1 and len(optarch_parts[0]) == 1
if not is_single_value and any(len(i) != 2 for i in optarch_parts):
raise EasyBuildError("The optarch option has an incorrect syntax: %s", self.options.optarch)

# if there are options for different compilers, we set up a dict
if is_single_value:
# if optarch is not in mapping format, we do nothing and just keep the string
self.log.info("Keeping optarch raw: %s", self.options.optarch)
else:
# if there are options for different compilers, we set up a dict
if OPTARCH_MAP_CHAR in optarch_parts[0]:
optarch_dict = {}
for compiler, compiler_opt in [p.split(OPTARCH_MAP_CHAR) for p in optarch_parts]:
if compiler in optarch_dict:
raise EasyBuildError("The optarch option contains duplicated entries for compiler %s: %s",
compiler, self.options.optarch)
optarch_dict = {}
for key, compiler_opt in optarch_parts:
# key can be either a compiler (only) or compiler and archspec(s)
key_parts = key.split(',')
compiler = key_parts.pop(0)
if key_parts:
for part, allowed_values in zip(key_parts, (CPU_ARCHITECTURES, CPU_FAMILIES, CPU_VECTOR_EXTS)):
if part not in allowed_values:
raise EasyBuildError("The optarch option has an incorrect syntax: %s\n"
"'%s' of '%s' is not in allowed values: %s",
self.options.optarch, part, key, allowed_values)
arch_specs = tuple(key_parts) if len(key_parts) > 1 else key_parts[0]
else:
arch_specs = None
try:
compiler_dict = optarch_dict[compiler]
# Convert single entry to dict if required
if not isinstance(compiler_dict, dict):
compiler_dict = {None: compiler_dict}
optarch_dict[compiler] = compiler_dict
if arch_specs in compiler_dict:
if arch_specs is None:
raise EasyBuildError("The optarch option contains a duplicated entry for compiler %s: %s",
compiler, self.options.optarch)
else:
raise EasyBuildError("The optarch option contains a duplicated entry %s for compiler %s: %s",
arch_specs, compiler, self.options.optarch)
else:
compiler_dict[arch_specs] = compiler_opt
except KeyError:
# Keep the dict flat when no archspecs are given
if arch_specs is None:
optarch_dict[compiler] = compiler_opt
self.options.optarch = optarch_dict
self.log.info("Transforming optarch into a dict: %s", self.options.optarch)
# if optarch is not in mapping format, we do nothing and just keep the string
else:
self.log.info("Keeping optarch raw: %s", self.options.optarch)
else:
optarch_dict[compiler] = {arch_specs: compiler_opt}
self.options.optarch = optarch_dict
self.log.info("Transforming optarch into a dict: %s", self.options.optarch)

def _postprocess_close_pr_reasons(self):
"""Postprocess --close-pr-reasons options"""
Expand Down
2 changes: 2 additions & 0 deletions easybuild/tools/systemtools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1131,6 +1131,8 @@ def create_possible_keys():
yield (arch, )
# Also allow single string entry
yield arch
# Default fallback for any arch
yield None

result = None
for key in create_possible_keys():
Expand Down
20 changes: 11 additions & 9 deletions easybuild/tools/toolchain/compiler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,7 @@ def _set_optimal_architecture(self, default_optarch=None):
:param default_optarch: default value to use for optarch, rather than using default value based on architecture
(--optarch and --optarch=GENERIC still override this value)
"""
ec_optarch = self.options.get('optarch', False)
ec_optarch = self.options.get('optarch')
if isinstance(ec_optarch, string_type):
if OPTARCH_MAP_CHAR in ec_optarch:
error_msg = "When setting optarch in the easyconfig (found %s), " % ec_optarch
Expand All @@ -346,20 +346,22 @@ def _set_optimal_architecture(self, default_optarch=None):
optarch = build_option('optarch')

# --optarch is specified with flags to use
if optarch is not None and isinstance(optarch, dict):
if isinstance(optarch, dict):
# optarch has been validated as complex string with multiple compilers and converted to a dictionary
# first try module names, then the family in optarch
current_compiler_names = (getattr(self, 'COMPILER_MODULE_NAME', []) +
[getattr(self, 'COMPILER_FAMILY', None)])
current_compiler_names = getattr(self, 'COMPILER_MODULE_NAME') or []
compiler_family = getattr(self, 'COMPILER_FAMILY')
if compiler_family:
current_compiler_names.append(compiler_family)
compiler_optarch = None
for current_compiler in current_compiler_names:
if current_compiler in optarch:
optarch = optarch[current_compiler]
compiler_optarch = optarch[current_compiler]
break
# still a dict: no option for this compiler
if isinstance(optarch, dict):
optarch = None
if compiler_optarch is None:
self.log.info("_set_optimal_architecture: no optarch found for compiler %s. Ignoring option.",
current_compiler)
current_compiler_names)
optarch = self._pick_optarch_entry(compiler_optarch)

use_generic = False
if optarch is not None:
Expand Down
27 changes: 25 additions & 2 deletions test/framework/options.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5226,10 +5226,14 @@ def test_parse_optarch(self):
options.options.optarch = 'Intel:something:somethingelse'
self.assertErrorRegex(EasyBuildError, error_msg, options.postprocess)

error_msg = "The optarch option contains duplicated entries for compiler"
error_msg = "The optarch option contains a duplicated entry for compiler"
options.options.optarch = 'Intel:something;GCC:somethingelse;Intel:anothersomething'
self.assertErrorRegex(EasyBuildError, error_msg, options.postprocess)

error_msg = "'x86' of 'x86' is not in allowed values"
options.options.optarch = 'Intel,x86:something'
self.assertErrorRegex(EasyBuildError, error_msg, options.postprocess)

# Check the parsing itself
gcc_generic_flags = "march=x86-64 -mtune=generic"
test_cases = [
Expand All @@ -5239,7 +5243,26 @@ def test_parse_optarch(self):
('Intel:xHost', {'Intel': 'xHost'}),
('Intel:GENERIC', {'Intel': 'GENERIC'}),
('Intel:xHost;GCC:%s' % gcc_generic_flags, {'Intel': 'xHost', 'GCC': gcc_generic_flags}),
('Intel:;GCC:%s' % gcc_generic_flags, {'Intel': '', 'GCC': gcc_generic_flags}),
# Allow empty values and spaces
('Intel:; GCC:%s' % gcc_generic_flags, {'Intel': '', 'GCC': gcc_generic_flags}),
# Arch specific values. With Compiler only selection at different places in the list
(
'Intel,x86_64,Intel,sse:i86iSSE; Intel,x86_64,AMD:i86AMD; Intel,POWER:iPwr; Intel:xHost; '
'GCC:GENERIC; Clang:cl; Clang,POWER:clPwr',
{
'Intel': {
('x86_64', 'Intel', 'sse'): 'i86iSSE',
('x86_64', 'AMD'): 'i86AMD',
'POWER': 'iPwr',
None: 'xHost',
},
'GCC': 'GENERIC',
'Clang': {
None: 'cl',
'POWER': 'clPwr',
},
},
),
]

for optarch_string, optarch_parsed in test_cases:
Expand Down
28 changes: 18 additions & 10 deletions test/framework/systemtools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -923,16 +923,24 @@ def test_pick_opt_arch(self):
avx = [st.SSE, st.SSE2, st.AVX]
avx2 = avx + [st.AVX2]
avx512 = avx2 + [st.AVX512F]
self.assertEqual(st.pick_opt_arch(options, st.X86_64, st.INTEL, avx2), 'opt-x86')
self.assertEqual(st.pick_opt_arch(options, st.X86_64, st.INTEL, []), 'opt-x86')
self.assertEqual(st.pick_opt_arch(options, st.X86_64, st.AMD, avx), 'opt-amd')
self.assertEqual(st.pick_opt_arch(options, st.X86_64, st.AMD, avx2), 'opt-amd-avx2')
self.assertEqual(st.pick_opt_arch(options, st.X86_64, st.AMD, avx512), 'opt-amd-avx2')
self.assertEqual(st.pick_opt_arch(options, st.AARCH64, st.AMD, avx2), 'opt-aarch64')
self.assertEqual(st.pick_opt_arch(options, st.AARCH64, st.INTEL, avx2), 'opt-aarch64-sse')
self.assertEqual(st.pick_opt_arch(options, st.AARCH64, st.INTEL, [st.SSE]), 'opt-aarch64-sse')
self.assertEqual(st.pick_opt_arch(options, st.AARCH32, st.AMD, avx2), None)
self.assertEqual(st.pick_opt_arch(options, st.POWER, st.IBM, []), None)
for include_none in (False, True):
if include_none:
# Special fallback option when no entry matches
options[None] = 'opt-none'
self.assertEqual(st.pick_opt_arch(options, st.X86_64, st.INTEL, avx2), 'opt-x86')
self.assertEqual(st.pick_opt_arch(options, st.X86_64, st.INTEL, []), 'opt-x86')
self.assertEqual(st.pick_opt_arch(options, st.X86_64, st.AMD, avx), 'opt-amd')
self.assertEqual(st.pick_opt_arch(options, st.X86_64, st.AMD, avx2), 'opt-amd-avx2')
self.assertEqual(st.pick_opt_arch(options, st.X86_64, st.AMD, avx512), 'opt-amd-avx2')
self.assertEqual(st.pick_opt_arch(options, st.AARCH64, st.AMD, avx2), 'opt-aarch64')
self.assertEqual(st.pick_opt_arch(options, st.AARCH64, st.INTEL, avx2), 'opt-aarch64-sse')
self.assertEqual(st.pick_opt_arch(options, st.AARCH64, st.INTEL, [st.SSE]), 'opt-aarch64-sse')
if include_none:
self.assertEqual(st.pick_opt_arch(options, st.AARCH32, st.AMD, avx2), 'opt-none')
self.assertEqual(st.pick_opt_arch(options, st.POWER, st.IBM, []), 'opt-none')
else:
self.assertEqual(st.pick_opt_arch(options, st.AARCH32, st.AMD, avx2), None)
self.assertEqual(st.pick_opt_arch(options, st.POWER, st.IBM, []), None)

def test_check_os_dependency(self):
"""Test check_os_dependency."""
Expand Down

0 comments on commit 6298aa9

Please sign in to comment.