diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index b79f615612..78f0344cda 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -219,6 +219,7 @@ def __init__(self, ec): # keep track of initial environment we start in, so we can restore it if needed self.initial_environ = copy.deepcopy(os.environ) + self.reset_environ = None self.tweaked_env_vars = {} # should we keep quiet? @@ -457,9 +458,7 @@ def fetch_extension_sources(self, skip_checksums=False): Find source file for extensions. """ exts_sources = [] - self.cfg.enable_templating = False - exts_list = self.cfg['exts_list'] - self.cfg.enable_templating = True + exts_list = self.cfg.get_ref('exts_list') if self.dry_run: self.dry_run_msg("\nList of sources/patches for extensions:") @@ -477,9 +476,7 @@ def fetch_extension_sources(self, skip_checksums=False): # make sure we grab *raw* dict of default options for extension, # since it may use template values like %(name)s & %(version)s - self.cfg.enable_templating = False - ext_options = copy.deepcopy(self.cfg['exts_default_options']) - self.cfg.enable_templating = True + ext_options = copy.deepcopy(self.cfg.get_ref('exts_default_options')) def_src_tmpl = "%(name)s-%(version)s.tar.gz" @@ -863,10 +860,25 @@ def make_builddir(self): # otherwise we wipe the already partially populated installation directory, # see https://github.com/easybuilders/easybuild-framework/issues/2556 if not (self.build_in_installdir and self.iter_idx > 0): + # make sure we no longer sit in the build directory before cleaning it. + change_dir(self.orig_workdir) self.make_dir(self.builddir, self.cfg['cleanupoldbuild']) trace_msg("build dir: %s" % self.builddir) + def reset_env(self): + """ + Reset environment. + When iterating over builddependencies, every time we start a new iteration + we need to restore the environment to where it was before the relevant modules + were loaded. + """ + env.reset_changes() + if self.reset_environ is None: + self.reset_environ = copy.deepcopy(os.environ) + else: + restore_env(self.reset_environ) + def gen_installdir(self): """ Generate the name of the installation directory. @@ -1371,10 +1383,8 @@ def skip_extensions(self): - use this to detect existing extensions and to remove them from self.exts - based on initial R version """ - # disabling templating is required here to support legacy string templates like name/version - self.cfg.enable_templating = False - exts_filter = self.cfg['exts_filter'] - self.cfg.enable_templating = True + # obtaining untemplated reference value is required here to support legacy string templates like name/version + exts_filter = self.cfg.get_ref('exts_filter') if not exts_filter or len(exts_filter) == 0: raise EasyBuildError("Skipping of extensions, but no exts_filter set in easyconfig") @@ -1471,21 +1481,25 @@ def handle_iterate_opts(self): """Handle options relevant during iterated part of build/install procedure.""" # disable templating in this function, since we're messing about with values in self.cfg + prev_enable_templating = self.cfg.enable_templating self.cfg.enable_templating = False - # handle configure/build/install options that are specified as lists + # tell easyconfig we are iterating (used by dependencies() and builddependencies()) + self.cfg.iterating = True + + # handle configure/build/install options that are specified as lists (+ perhaps builddependencies) # set first element to be used, keep track of list in self.iter_opts - # this will only be done during first iteration, since after that the options won't be lists anymore - for opt in ITERATE_OPTIONS: + # only needs to be done during first iteration, since after that the options won't be lists anymore + if self.iter_idx == 0: # keep track of list, supply first element as first option to handle - if isinstance(self.cfg[opt], (list, tuple)): + for opt in self.cfg.iterate_options: self.iter_opts[opt] = self.cfg[opt] # copy self.log.debug("Found list for %s: %s", opt, self.iter_opts[opt]) if self.iter_opts: self.log.info("Current iteration index: %s", self.iter_idx) - # pop first element from all *_list options as next value to use + # pop first element from all iterative easyconfig parameters as next value to use for opt in self.iter_opts: if len(self.iter_opts[opt]) > self.iter_idx: self.cfg[opt] = self.iter_opts[opt][self.iter_idx] @@ -1494,7 +1508,7 @@ def handle_iterate_opts(self): self.log.debug("Next value for %s: %s" % (opt, str(self.cfg[opt]))) # re-enable templating before self.cfg values are used - self.cfg.enable_templating = True + self.cfg.enable_templating = prev_enable_templating # prepare for next iteration (if any) self.iter_idx += 1 @@ -1502,14 +1516,18 @@ def handle_iterate_opts(self): def restore_iterate_opts(self): """Restore options that were iterated over""" # disable templating, since we're messing about with values in self.cfg + prev_enable_templating = self.cfg.enable_templating self.cfg.enable_templating = False for opt in self.iter_opts: self.cfg[opt] = self.iter_opts[opt] self.log.debug("Restored value of '%s' that was iterated over: %s", opt, self.cfg[opt]) + # tell easyconfig we are no longer iterating (used by dependencies() and builddependencies()) + self.cfg.iterating = False + # re-enable templating before self.cfg values are used - self.cfg.enable_templating = True + self.cfg.enable_templating = prev_enable_templating def det_iter_cnt(self): """Determine iteration count based on configure/build/install options that may be lists.""" @@ -1822,18 +1840,20 @@ def prepare_step(self, start_dir=True): # list of paths to include in RPATH filter; # only include builddir if we're not building in installation directory - self.rpath_filter_dirs.append(tempfile.gettempdir()) + self.rpath_filter_dirs = [tempfile.gettempdir()] if not self.build_in_installdir: self.rpath_filter_dirs.append(self.builddir) # always include '/lib', '/lib64', $ORIGIN, $ORIGIN/../lib and $ORIGIN/../lib64 # $ORIGIN will be resolved by the loader to be the full path to the executable or shared object # see also https://linux.die.net/man/8/ld-linux; - self.rpath_include_dirs.append(os.path.join(self.installdir, 'lib')) - self.rpath_include_dirs.append(os.path.join(self.installdir, 'lib64')) - self.rpath_include_dirs.append('$ORIGIN') - self.rpath_include_dirs.append('$ORIGIN/../lib') - self.rpath_include_dirs.append('$ORIGIN/../lib64') + self.rpath_include_dirs = [ + os.path.join(self.installdir, 'lib'), + os.path.join(self.installdir, 'lib64'), + '$ORIGIN', + '$ORIGIN/../lib', + '$ORIGIN/../lib64', + ] # prepare toolchain: load toolchain module and dependencies, set up build environment self.toolchain.prepare(self.cfg['onlytcmod'], deps=self.cfg.dependencies(), silent=self.silent, @@ -1841,6 +1861,7 @@ def prepare_step(self, start_dir=True): # keep track of environment variables that were tweaked and need to be restored after environment got reset # $TMPDIR may be tweaked for OpenMPI 2.x, which doesn't like long $TMPDIR paths... + self.tweaked_env_vars = {} for var in ['TMPDIR']: if os.environ.get(var) != self.initial_environ.get(var): self.tweaked_env_vars[var] = os.environ.get(var) @@ -2660,7 +2681,7 @@ def get_step(tag, descr, substeps, skippable, initial=True): ready_substeps = [ (False, lambda x: x.check_readiness_step), (True, lambda x: x.make_builddir), - (True, lambda x: env.reset_changes), + (True, lambda x: x.reset_env), (True, lambda x: x.handle_iterate_opts), ] @@ -2678,14 +2699,6 @@ def source_step_spec(initial): """Return source step specified.""" return get_step(SOURCE_STEP, "unpacking", source_substeps, True, initial=initial) - def prepare_step_spec(initial): - """Return prepare step specification.""" - if initial: - substeps = [lambda x: x.prepare_step] - else: - substeps = [lambda x: x.guess_start_dir] - return (PREPARE_STEP, 'preparing', substeps, False) - install_substeps = [ (False, lambda x: x.stage_install_step), (False, lambda x: x.make_installdir), @@ -2696,10 +2709,11 @@ def install_step_spec(initial): """Return install step specification.""" return get_step(INSTALL_STEP, "installing", install_substeps, True, initial=initial) - # format for step specifications: (stop_name: (description, list of functions, skippable)) + # format for step specifications: (step_name, description, list of functions, skippable) # core steps that are part of the iterated loop patch_step_spec = (PATCH_STEP, 'patching', [lambda x: x.patch_step], True) + prepare_step_spec = (PREPARE_STEP, 'preparing', [lambda x: x.prepare_step], False) configure_step_spec = (CONFIGURE_STEP, 'configuring', [lambda x: x.configure_step], True) build_step_spec = (BUILD_STEP, 'building', [lambda x: x.build_step], True) test_step_spec = (TEST_STEP, 'testing', [lambda x: x.test_step], True) @@ -2710,7 +2724,7 @@ def install_step_spec(initial): ready_step_spec(True), source_step_spec(True), patch_step_spec, - prepare_step_spec(True), + prepare_step_spec, configure_step_spec, build_step_spec, test_step_spec, @@ -2723,7 +2737,7 @@ def install_step_spec(initial): ready_step_spec(False), source_step_spec(False), patch_step_spec, - prepare_step_spec(False), + prepare_step_spec, configure_step_spec, build_step_spec, test_step_spec, @@ -3270,9 +3284,7 @@ def make_checksum_lines(checksums, indent_level): # make sure we grab *raw* dict of default options for extension, # since it may use template values like %(name)s & %(version)s - app.cfg.enable_templating = False - exts_default_options = app.cfg['exts_default_options'] - app.cfg.enable_templating = True + exts_default_options = app.cfg.get_ref('exts_default_options') for key, val in sorted(ext_options.items()): if key != 'checksums' and val != exts_default_options.get(key): diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 2c19b0198b..53517f4680 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -73,7 +73,7 @@ from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME, DUMMY_TOOLCHAIN_VERSION from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES, TOOLCHAIN_CAPABILITY_CUDA from easybuild.tools.toolchain.utilities import get_toolchain, search_toolchain -from easybuild.tools.utilities import quote_py_str, remove_unwanted_chars +from easybuild.tools.utilities import flatten, quote_py_str, remove_unwanted_chars from easybuild.tools.version import VERSION from easybuild.toolchains.compiler.cuda import Cuda @@ -83,7 +83,8 @@ MANDATORY_PARAMS = ['name', 'version', 'homepage', 'description', 'toolchain'] # set of configure/build/install options that can be provided as lists for an iterated build -ITERATE_OPTIONS = ['preconfigopts', 'configopts', 'prebuildopts', 'buildopts', 'preinstallopts', 'installopts'] +ITERATE_OPTIONS = ['builddependencies', + 'preconfigopts', 'configopts', 'prebuildopts', 'buildopts', 'preinstallopts', 'installopts'] # name of easyconfigs archive subdirectory EASYCONFIGS_ARCHIVE_DIR = '__archive__' @@ -539,13 +540,28 @@ def parse(self): run_hook(PARSE, hooks, args=[self], msg=parse_hook_msg) + # create a list of all options that are actually going to be iterated over + # builddependencies are always a list, need to look deeper down below + self.iterate_options = [opt for opt in ITERATE_OPTIONS + if opt != 'builddependencies' and isinstance(self[opt], (list, tuple))] + self.iterating = False + # parse dependency specifications # it's important that templating is still disabled at this stage! self.log.info("Parsing dependency specifications...") - self['builddependencies'] = [self._parse_dependency(dep, build_only=True) for dep in self['builddependencies']] self['dependencies'] = [self._parse_dependency(dep) for dep in self['dependencies']] self['hiddendependencies'] = [self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies']] + # need to take into account that builddependencies may need to be iterated over, + # i.e. when the value is a list of lists of tuples + builddeps = self['builddependencies'] + if builddeps and all(isinstance(x, (list, tuple)) for b in builddeps for x in b): + self.iterate_options.append('builddependencies') + builddeps = [[self._parse_dependency(dep, build_only=True) for dep in x] for x in builddeps] + else: + builddeps = [self._parse_dependency(dep, build_only=True) for dep in builddeps] + self['builddependencies'] = builddeps + # restore templating self.enable_templating = prev_enable_templating @@ -663,15 +679,14 @@ def validate_iterate_opts_lists(self): # when lists are used, they should be all of same length # list of length 1 are treated as if it were strings in EasyBlock opt_counts = [] - for opt in ITERATE_OPTIONS: + for opt in self.iterate_options: # anticipate changes in available easyconfig parameters (e.g. makeopts -> buildopts?) if self.get(opt, None) is None: raise EasyBuildError("%s not available in self.cfg (anymore)?!", opt) # keep track of list, supply first element as first option to handle - if isinstance(self[opt], (list, tuple)): - opt_counts.append((opt, len(self[opt]))) + opt_counts.append((opt, len(self[opt]))) # make sure that options that specify lists have the same length list_opt_lengths = [length for (opt, length) in opt_counts if length > 1] @@ -686,29 +701,33 @@ def filter_hidden_deps(self): """ faulty_deps = [] - for hidden_idx, hidden_dep in enumerate(self['hiddendependencies']): + # obtain reference to original lists, so their elements can be changed in place + deps = dict([(key, self.get_ref(key)) for key in ['dependencies', 'builddependencies', 'hiddendependencies']]) + + if 'builddependencies' in self.iterate_options: + deplists = deps['builddependencies'] + else: + deplists = [deps['builddependencies']] + + deplists.append(deps['dependencies']) + + for hidden_idx, hidden_dep in enumerate(deps['hiddendependencies']): hidden_mod_name = ActiveMNS().det_full_module_name(hidden_dep) visible_mod_name = ActiveMNS().det_full_module_name(hidden_dep, force_visible=True) # replace (build) dependencies with their equivalent hidden (build) dependency (if any) replaced = False - for key in ['builddependencies', 'dependencies']: - for idx, dep in enumerate(self[key]): + for deplist in deplists: + for idx, dep in enumerate(deplist): dep_mod_name = dep['full_mod_name'] if dep_mod_name in [visible_mod_name, hidden_mod_name]: - # templating must be temporarily disabled to obtain reference to original lists, - # so their elements can be changed in place - # see comments in resolve_template - enable_templating = self.enable_templating - self.enable_templating = False # track whether this hidden dep is listed as a build dep - orig_hidden_dep = self['hiddendependencies'][hidden_idx] - if key == 'builddependencies': - orig_hidden_dep['build_only'] = True + hidden_dep = deps['hiddendependencies'][hidden_idx] + hidden_dep['build_only'] = dep['build_only'] + # actual replacement - self[key][idx] = orig_hidden_dep - self.enable_templating = enable_templating + deplist[idx] = hidden_dep replaced = True if dep_mod_name == visible_mod_name: @@ -818,13 +837,14 @@ def dependencies(self, build_only=False): """ Returns an array of parsed dependencies (after filtering, if requested) dependency = {'name': '', 'version': '', 'dummy': (False|True), 'versionsuffix': '', 'toolchain': ''} + Iterable builddependencies are flattened when not iterating. :param build_only: only return build dependencies, discard others """ if build_only: - deps = self['builddependencies'] + deps = self.builddependencies() else: - deps = self['dependencies'] + self['builddependencies'] + deps = self['dependencies'] + self.builddependencies() # if filter-deps option is provided we "clean" the list of dependencies for # each processed easyconfig to remove the unwanted dependencies @@ -837,9 +857,22 @@ def dependencies(self, build_only=False): def builddependencies(self): """ - return the parsed build dependencies + Return a flat list of the parsed build dependencies + When builddependencies are iterable they are flattened lists with + duplicates removed outside of the iterating process, because the callers + want simple lists. """ - return self['builddependencies'] + builddeps = self['builddependencies'] + + if 'builddependencies' in self.iterate_options and not self.iterating: + # flatten and remove duplicates (can't use 'nub', since dict values are not hashable) + all_builddeps = flatten(builddeps) + builddeps = [] + for dep in all_builddeps: + if dep not in builddeps: + builddeps.append(dep) + + return builddeps @property def name(self): @@ -1110,11 +1143,20 @@ def _finalize_dependencies(self): for key in DEPENDENCY_PARAMETERS: # loop over a *copy* of dependency dicts (with resolved templates); - # to update the original dep dict, we need to index with idx into self._config[key][0]... - for idx, dep in enumerate(self[key]): + deps = self[key] + + # to update the original dep dict, we need to get a reference with templating disabled... + deps_ref = self.get_ref(key) + + # take into account that this *dependencies parameter may be iterated over + if key in self.iterate_options: + deps = flatten(deps) + deps_ref = flatten(deps_ref) + + for idx, dep in enumerate(deps): # reference to original dep dict, this is the one we should be updating - orig_dep = self._config[key][0][idx] + orig_dep = deps_ref[idx] if filter_deps and orig_dep['name'] in filter_deps: self.log.debug("Skipping filtered dependency %s when finalising dependencies", orig_dep['name']) @@ -1221,6 +1263,24 @@ def __getitem__(self, key): return value + def get_ref(self, key): + """ + Obtain reference to original/untemplated value of specified easyconfig parameter + rather than a copied value with templated values. + """ + # see also comments in resolve_template + + # temporarily disable templating + prev_enable_templating = self.enable_templating + self.enable_templating = False + + ref = self[key] + + # restore previous value for 'enable_templating' + self.enable_templating = prev_enable_templating + + return ref + @handle_deprecated_or_replaced_easyconfig_parameters def __setitem__(self, key, value): """Set value of specified easyconfig parameter (help text & co is left untouched)""" diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index 25e2266593..7af9528f82 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -30,7 +30,6 @@ :author: Stijn De Weirdt (Ghent University) :author: Kenneth Hoste (Ghent University) """ -import copy import os import re import tempfile @@ -52,6 +51,7 @@ # dependency parameters always need to be reformatted, to correctly deal with dumping parsed dependencies REFORMAT_FORCED_PARAMS = ['sanity_check_paths'] + DEPENDENCY_PARAMETERS REFORMAT_SKIPPED_PARAMS = ['toolchain', 'toolchainopts'] +REFORMAT_LIST_OF_LISTS_OF_TUPLES = ['builddependencies'] REFORMAT_THRESHOLD_LENGTH = 100 # only reformat lines that would be longer than this amount of characters REFORMAT_ORDERED_ITEM_KEYS = { 'sanity_check_paths': ['files', 'dirs'], @@ -140,6 +140,7 @@ def _reformat_line(self, param_name, param_val, outer=False, addlen=0): # note: this does not take into account the parameter name + '=', only the value line_too_long = len(param_strval) + addlen > REFORMAT_THRESHOLD_LENGTH forced = param_name in REFORMAT_FORCED_PARAMS + list_of_lists_of_tuples_param = param_name in REFORMAT_LIST_OF_LISTS_OF_TUPLES if param_name in REFORMAT_SKIPPED_PARAMS: self.log.info("Skipping reformatting value for parameter '%s'", param_name) @@ -170,9 +171,17 @@ def _reformat_line(self, param_name, param_val, outer=False, addlen=0): for item in param_val: comment = self._get_item_comments(param_name, item).get(str(item), '') addlen = addlen + len(INDENT_4SPACES) + len(comment) + is_list_of_lists_of_tuples = isinstance(item, list) and all(isinstance(x, tuple) for x in item) + if list_of_lists_of_tuples_param and is_list_of_lists_of_tuples: + itemstr = '[' + (',\n ' + INDENT_4SPACES).join([ + self._reformat_line(param_name, subitem, outer=True, addlen=addlen) + for subitem in item]) + ']' + else: + itemstr = self._reformat_line(param_name, item, addlen=addlen) + res += item_tmpl % { 'comment': comment, - 'item': self._reformat_line(param_name, item, addlen=addlen) + 'item': itemstr } # end with closing character: ], ), } @@ -227,9 +236,13 @@ def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_ for key in group: val = ecfg[key] if val != default_values[key]: - # dependency easyconfig parameters were parsed, so these need special care to 'unparse' them + # dependency easyconfig parameters were parsed, so these need special care to 'unparse' them; + # take into account that these parameters may be iterative (i.e. a list of lists of parsed deps) if key in DEPENDENCY_PARAMETERS: - valstr = [dump_dependency(d, ecfg['toolchain']) for d in val] + if key in ecfg.iterate_options: + valstr = [[dump_dependency(d, ecfg['toolchain']) for d in dep] for dep in val] + else: + valstr = [dump_dependency(d, ecfg['toolchain']) for d in val] elif key == 'toolchain': valstr = "{'name': '%(name)s', 'version': '%(version)s'}" % ecfg[key] else: diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index adc7a3144b..694907700a 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -56,7 +56,7 @@ from easybuild.tools.robot import resolve_dependencies, robot_find_easyconfig from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES -from easybuild.tools.utilities import quote_str +from easybuild.tools.utilities import flatten, quote_str _log = fancylogger.getLogger('easyconfig.tweak', fname=False) @@ -821,21 +821,29 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= :return: Location of the modified easyconfig file """ # Fully parse the original easyconfig - parsed_ec = process_easyconfig(ec_spec, validate=False)[0] + parsed_ec = process_easyconfig(ec_spec, validate=False)[0]['ec'] + # Replace the toolchain if the mapping exists - tc_name = parsed_ec['ec']['toolchain']['name'] + tc_name = parsed_ec['toolchain']['name'] if tc_name in toolchain_mapping: new_toolchain = toolchain_mapping[tc_name] - _log.debug("Replacing parent toolchain %s with %s", parsed_ec['ec']['toolchain'], new_toolchain) - parsed_ec['ec']['toolchain'] = new_toolchain + _log.debug("Replacing parent toolchain %s with %s", parsed_ec['toolchain'], new_toolchain) + parsed_ec['toolchain'] = new_toolchain # Replace the toolchains of all the dependencies for key in DEPENDENCY_PARAMETERS: # loop over a *copy* of dependency dicts (with resolved templates); - # to update the original dep dict, we need to index with idx into self._config[key][0]... - for idx, dep in enumerate(parsed_ec['ec'][key]): + # to update the original dep dict, we need to get a reference with templating disabled... + val = parsed_ec[key] + orig_val = parsed_ec.get_ref(key) + + if key in parsed_ec.iterate_options: + val = flatten(val) + orig_val = flatten(orig_val) + + for idx, dep in enumerate(val): # reference to original dep dict, this is the one we should be updating - orig_dep = parsed_ec['ec']._config[key][0][idx] + orig_dep = orig_val[idx] # skip dependencies that are marked as external modules if dep['external_module']: continue @@ -849,10 +857,10 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name(dep) orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(dep) # Determine the name of the modified easyconfig and dump it to target_dir - ec_filename = '%s-%s.eb' % (parsed_ec['ec']['name'], det_full_ec_version(parsed_ec['ec'])) + ec_filename = '%s-%s.eb' % (parsed_ec['name'], det_full_ec_version(parsed_ec)) tweaked_spec = os.path.join(targetdir or tempfile.gettempdir(), ec_filename) - parsed_ec['ec'].dump(tweaked_spec, always_overwrite=False, backup=True) + parsed_ec.dump(tweaked_spec, always_overwrite=False, backup=True) _log.debug("Dumped easyconfig tweaked via --try-toolchain* to %s", tweaked_spec) return tweaked_spec diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index a5bc66f118..f4691e4fe7 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -2465,6 +2465,38 @@ def test_filename(self): ec = EasyConfig(test_ec) self.assertTrue(ec.filename(), os.path.basename(test_ec)) + def test_get_ref(self): + """Test get_ref method.""" + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-iter.eb')) + + # without using get_ref, we get a (templated) copy rather than the original value + sources = ec['sources'] + self.assertEqual(sources, ['toy-0.0.tar.gz']) + self.assertFalse(sources is ec._config['sources'][0]) + + # same for .get + sources = ec.get('sources') + self.assertEqual(sources, ['toy-0.0.tar.gz']) + self.assertFalse(sources is ec._config['sources'][0]) + + # with get_ref, we get the original untemplated value + sources_ref = ec.get_ref('sources') + self.assertEqual(sources_ref, ['%(name)s-%(version)s.tar.gz']) + self.assertTrue(sources_ref is ec._config['sources'][0]) + + sanity_check_paths_ref = ec.get_ref('sanity_check_paths') + self.assertTrue(sanity_check_paths_ref is ec._config['sanity_check_paths'][0]) + + # also items inside are still references to original (i.e. not copies) + self.assertTrue(sanity_check_paths_ref['files'] is ec._config['sanity_check_paths'][0]['files']) + + # get_ref also works for values other than lists/dicts + self.assertEqual(ec['description'], "Toy C program, 100% toy.") + descr_ref = ec.get_ref('description') + self.assertEqual(descr_ref, "Toy C program, 100% %(name)s.") + self.assertTrue(descr_ref is ec._config['description'][0]) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb index fe6e2840f5..085768f707 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb @@ -3,21 +3,28 @@ version = '0.0' versionsuffix = '-iter' homepage = 'https://easybuilders.github.io/easybuild' -description = "Toy C program, 100% toy." +description = "Toy C program, 100% %(name)s." -toolchain = {'name': 'dummy', 'version': 'dummy'} +toolchain = {'name': 'dummy', 'version': ''} sources = [SOURCE_TAR_GZ] patches = ['toy-0.0_fix-silly-typo-in-printf-statement.patch'] +builddependencies = [ + [('GCC', '6.4.0-2.28'), + ('OpenMPI', '2.1.2', '', ('GCC', '6.4.0-2.28'))], + [('GCC', '6.4.0-2.28')], + [('GCC', '7.3.0-2.30')], +] + buildopts = [ '', - "-O2; mv %(name)s toy_O2", - "-O1; mv %(name)s toy_O1", + "-O2; mv %(name)s toy_O2_$EBVERSIONGCC", + "-O1; mv %(name)s toy_O1_$EBVERSIONGCC", ] sanity_check_paths = { - 'files': [('bin/yot', 'bin/toy'), 'bin/toy_O1', 'bin/toy_O2'], + 'files': [('bin/yot', 'bin/toy'), 'bin/toy_O1_7.3.0-2.30', 'bin/toy_O2_6.4.0-2.28'], 'dirs': ['bin'], } diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index e2b8b953f3..546439af0d 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1822,7 +1822,7 @@ def test_toy_iter(self): topdir = os.path.abspath(os.path.dirname(__file__)) toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-iter.eb') - expected_buildopts = ['', '-O2; mv %(name)s toy_O2', '-O1; mv %(name)s toy_O1'] + expected_buildopts = ['', '-O2; mv %(name)s toy_O2_$EBVERSIONGCC', '-O1; mv %(name)s toy_O1_$EBVERSIONGCC'] for extra_args in [None, ['--minimal-toolchains']]: # sanity check will make sure all entries in buildopts list were taken into account