Skip to content

Commit

Permalink
Remove _parallelLegacy
Browse files Browse the repository at this point in the history
  • Loading branch information
Flamefire committed Sep 17, 2024
1 parent 9e11b40 commit a793f08
Show file tree
Hide file tree
Showing 4 changed files with 28 additions and 101 deletions.
10 changes: 0 additions & 10 deletions easybuild/framework/easyblock.py
Original file line number Diff line number Diff line change
Expand Up @@ -2252,16 +2252,6 @@ def set_parallel(self):
if par is not None:
self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par)

# Transitional only in case some easyblocks still set/change cfg['parallel']
# Use _parallelLegacy to avoid deprecation warnings
# Remove for EasyBuild 5.0
cfg_par = self.cfg['_parallelLegacy']
if cfg_par is not None:
if par is None:
par = cfg_par
else:
par = min(int(par), int(cfg_par))

par = det_parallelism(par, maxpar=self.cfg['maxparallel'])
self.log.info("Setting parallelism: %s" % par)
self.cfg.parallel = par
Expand Down
23 changes: 4 additions & 19 deletions easybuild/framework/easyconfig/easyconfig.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,19 +120,15 @@ def handle_deprecated_or_replaced_easyconfig_parameters(ec_method):
def new_ec_method(self, key, *args, **kwargs):
"""Check whether any replace easyconfig parameters are still used"""
# map deprecated parameters to their replacements, issue deprecation warning(/error)
if key == 'parallel':
_log.deprecated("Easyconfig parameter 'parallel' is deprecated, "
"use 'maxparallel' or the parallel property instead.", '5.1')
# Use a "hidden" property for now to match behavior as closely as possible
key = '_parallelLegacy'
elif key in ALTERNATIVE_EASYCONFIG_PARAMETERS:
if key in ALTERNATIVE_EASYCONFIG_PARAMETERS:
key = ALTERNATIVE_EASYCONFIG_PARAMETERS[key]
elif key in DEPRECATED_EASYCONFIG_PARAMETERS:
depr_key = key
key, ver = DEPRECATED_EASYCONFIG_PARAMETERS[depr_key]
_log.deprecated("Easyconfig parameter '%s' is deprecated, use '%s' instead" % (depr_key, key), ver)
elif key in REPLACED_PARAMETERS:
_log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0')
newkey, ver = REPLACED_PARAMETERS[key]
_log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, newkey), ver)
return ec_method(self, key, *args, **kwargs)

return new_ec_method
Expand All @@ -149,9 +145,7 @@ def is_local_var_name(name):
"""
res = False
if name.startswith(LOCAL_VAR_PREFIX) or name.startswith('_'):
# Remove with EasyBuild 5.1
if name != '_parallelLegacy':
res = True
res = True
# __builtins__ is always defined as a 'local' variables
# single-letter local variable names are allowed (mainly for use in list comprehensions)
# in Python 2, variables defined in list comprehensions leak to the outside (no longer the case in Python 3)
Expand Down Expand Up @@ -508,8 +502,6 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi

# Storage for parallel property. Mark as unset initially
self._parallel = None
# Legacy value, remove with EasyBuild 5.1
self._config['_parallelLegacy'] = [None, '', ('', )]

# parse easyconfig file
self.build_specs = build_specs
Expand Down Expand Up @@ -711,11 +703,6 @@ def parse(self):
if missing_mandatory_keys:
raise EasyBuildError("mandatory parameters not provided in %s: %s", self.path, missing_mandatory_keys)

if 'parallel' in ec_vars:
# Replace value and issue better warning for EC params (as opposed to warnings meant for easyblocks)
self.log.deprecated("Easyconfig parameter 'parallel' is deprecated, use 'maxparallel' instead.", '5.1')
ec_vars['_parallelLegacy'] = ec_vars.pop('parallel')

# provide suggestions for typos. Local variable names are excluded from this check
possible_typos = [(key, difflib.get_close_matches(key.lower(), self._config.keys(), 1, 0.85))
for key in ec_vars if not is_local_var_name(key) and key not in self]
Expand Down Expand Up @@ -1235,8 +1222,6 @@ def parallel(self, value):
# Update backstorage and template value
self._parallel = value
self.template_values['parallel'] = value
# Backwards compat only. Remove with EasyBuild 5.1
self._config['_parallelLegacy'][0] = value

def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=False):
"""
Expand Down
8 changes: 5 additions & 3 deletions easybuild/framework/easyconfig/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,9 +107,11 @@

# replaced easyconfig parameters, and their replacements
REPLACED_PARAMETERS = {
'license': 'license_file',
'makeopts': 'buildopts',
'premakeopts': 'prebuildopts',
# <old_param>: (<new_param>, <removed_in_version>),
'license': ('license_file', '2.0'),
'makeopts': ('buildopts', '2.0'),
'parallel': ('maxparallel', '5.0'),
'premakeopts': ('prebuildopts', '2.0'),
}

_log = fancylogger.getLogger('easyconfig.parser', fname=False)
Expand Down
88 changes: 19 additions & 69 deletions test/framework/easyblock.py
Original file line number Diff line number Diff line change
Expand Up @@ -2199,30 +2199,31 @@ def test_parallel(self):
toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
toytxt = read_file(toy_ec)

handle, toy_ec_error = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb')
os.close(handle)
write_file(toy_ec_error, toytxt + "\nparallel = 123")

handle, toy_ec1 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb')
os.close(handle)
write_file(toy_ec1, toytxt + "\nparallel = 13")
write_file(toy_ec1, toytxt + "\nmaxparallel = None")

handle, toy_ec2 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb')
os.close(handle)
write_file(toy_ec2, toytxt + "\nparallel = 12\nmaxparallel = 6")
write_file(toy_ec2, toytxt + "\nmaxparallel = 6")

handle, toy_ec3 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb')
os.close(handle)
write_file(toy_ec3, toytxt + "\nparallel = False")

handle, toy_ec4 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb')
os.close(handle)
write_file(toy_ec4, toytxt + "\nmaxparallel = 6")

handle, toy_ec5 = tempfile.mkstemp(prefix='easyblock_test_file_', suffix='.eb')
os.close(handle)
write_file(toy_ec5, toytxt + "\nmaxparallel = False")
write_file(toy_ec3, toytxt + "\nmaxparallel = False")

import easybuild.tools.systemtools as st
auto_parallel = 15
st.det_parallelism._default_parallelism = auto_parallel

# 'parallel' easyconfig parameter specified is an error
self.assertRaises(EasyBuildError, EasyConfig, toy_ec_error)
self.assertErrorRegex(EasyBuildError, "Easyconfig parameter 'parallel' is replaced by 'maxparallel'",
EasyConfig, toy_ec_error)

# default: parallelism is derived from # available cores + ulimit
# Note that maxparallel has a default of 16, so we need a lower auto_paralle value here
test_eb = EasyBlock(EasyConfig(toy_ec))
Expand All @@ -2232,87 +2233,36 @@ def test_parallel(self):
auto_parallel = 128 # Don't limit by available CPU cores mock
st.det_parallelism._default_parallelism = auto_parallel

# only 'parallel' easyconfig parameter specified (no 'parallel' build option)
with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
test_eb = EasyBlock(EasyConfig(toy_ec1))
test_eb.check_readiness_step()
self.assertEqual(test_eb.cfg.parallel, 13)
with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
self.assertEqual(test_eb.cfg['parallel'], 13)

# both 'parallel' and 'maxparallel' easyconfig parameters specified (no 'parallel' build option)
with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
test_eb = EasyBlock(EasyConfig(toy_ec2))
test_eb.check_readiness_step()
self.assertEqual(test_eb.cfg.parallel, 6)
with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
self.assertEqual(test_eb.cfg['parallel'], 6)

# make sure 'parallel = False' is not overriden (no 'parallel' build option)
with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
test_eb = EasyBlock(EasyConfig(toy_ec3))
test_eb.check_readiness_step()
self.assertEqual(test_eb.cfg.parallel, False)
with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
self.assertEqual(test_eb.cfg['parallel'], False)

# only 'maxparallel' easyconfig parameter specified (no 'parallel' build option)
test_eb = EasyBlock(EasyConfig(toy_ec4))
test_eb = EasyBlock(EasyConfig(toy_ec2))
test_eb.check_readiness_step()
self.assertEqual(test_eb.cfg.parallel, 6)
with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
self.assertEqual(test_eb.cfg['parallel'], 6)

# make sure 'maxparallel = False' is treated as 1 (no 'parallel' build option)
test_eb = EasyBlock(EasyConfig(toy_ec5))
test_eb = EasyBlock(EasyConfig(toy_ec3))
test_eb.check_readiness_step()
self.assertEqual(test_eb.cfg.parallel, 1)
with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
self.assertEqual(test_eb.cfg['parallel'], 1)

# only 'parallel' build option specified
init_config(build_options={'parallel': '13', 'validate': False})
test_eb = EasyBlock(EasyConfig(toy_ec))
test_eb.check_readiness_step()
self.assertEqual(test_eb.cfg.parallel, 13)
with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
self.assertEqual(test_eb.cfg['parallel'], 13)

# both 'parallel' build option and easyconfig parameter specified (no 'maxparallel')
with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
test_eb = EasyBlock(EasyConfig(toy_ec1))
# 'maxparallel = None' is the same as unset
test_eb = EasyBlock(EasyConfig(toy_ec1))
test_eb.check_readiness_step()
self.assertEqual(test_eb.cfg.parallel, 13)
with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
self.assertEqual(test_eb.cfg['parallel'], 13)

# both 'parallel' and 'maxparallel' easyconfig parameters specified + 'parallel' build option
with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
test_eb = EasyBlock(EasyConfig(toy_ec2))
test_eb.check_readiness_step()
self.assertEqual(test_eb.cfg.parallel, 6)

# make sure 'parallel = False' is not overriden (with 'parallel' build option)
with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
test_eb = EasyBlock(EasyConfig(toy_ec3))
test_eb.check_readiness_step()
self.assertEqual(test_eb.cfg.parallel, 0)
with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
self.assertEqual(test_eb.cfg['parallel'], 0)

# only 'maxparallel' easyconfig parameter specified (with 'parallel' build option)
test_eb = EasyBlock(EasyConfig(toy_ec4))
# 'maxparallel' easyconfig parameter with 'parallel' build option
test_eb = EasyBlock(EasyConfig(toy_ec2))
test_eb.check_readiness_step()
self.assertEqual(test_eb.cfg.parallel, 6)
with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
self.assertEqual(test_eb.cfg['parallel'], 6)

# make sure 'maxparallel = False' is treated as 1 (with 'parallel' build option)
test_eb = EasyBlock(EasyConfig(toy_ec5))
test_eb = EasyBlock(EasyConfig(toy_ec3))
test_eb.check_readiness_step()
self.assertEqual(test_eb.cfg.parallel, 1)
with self.temporarily_allow_deprecated_behaviour(), self.mocked_stdout_stderr():
self.assertEqual(test_eb.cfg['parallel'], 1)

# Template updated correctly
test_eb.cfg['buildopts'] = '-j %(parallel)s'
Expand Down

0 comments on commit a793f08

Please sign in to comment.