From 08adff3185fadeaeb7367dc5f6084399f735de15 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 30 Jan 2019 20:31:55 +0000 Subject: [PATCH 01/28] Add iterate_builddependencies easyconfig option. This is a list of list of buildependencies. It gets iterated on top of the common builddependencies. This will allow e.g. to build with multiple Python versions. A toy test is included. The "prepare" step is now fully iterated which needed a few adjustments. Hopefully this doesn't break anything. --- easybuild/framework/easyblock.py | 39 +++++++++++-------- easybuild/framework/easyconfig/default.py | 1 + easybuild/framework/easyconfig/easyconfig.py | 14 +++++-- .../framework/easyconfig/format/format.py | 2 +- easybuild/framework/easyconfig/format/one.py | 12 +++++- .../test_ecs/t/toy/toy-0.0-iter.eb | 15 +++++-- test/framework/toy_build.py | 2 +- 7 files changed, 57 insertions(+), 28 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 34d0ba8e28..0fa13a0de4 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -863,6 +863,8 @@ def make_builddir(self): # otherwise we wipe the already partially populated installation directory, # see https://github.com/easybuilders/easybuild-framework/issues/2556 if not (self.build_in_installdir and self.iter_idx > 0): + # make sure we no longer sit in the build directory before cleaning it. + change_dir(self.orig_workdir) self.make_dir(self.builddir, self.cfg['cleanupoldbuild']) trace_msg("build dir: %s" % self.builddir) @@ -1478,7 +1480,11 @@ def handle_iterate_opts(self): # this will only be done during first iteration, since after that the options won't be lists anymore for opt in ITERATE_OPTIONS: # keep track of list, supply first element as first option to handle - if isinstance(self.cfg[opt], (list, tuple)): + if isinstance(self.cfg[opt], (list, tuple)) and self.cfg[opt] != [[]]: + if opt.startswith('iterate_') and self.iter_idx == 0: + # save original common value (e.g. builddependencies) + commonopt = opt[len('iterate_'):] + self.iter_opts[commonopt] = self.cfg[commonopt] self.iter_opts[opt] = self.cfg[opt] # copy self.log.debug("Found list for %s: %s", opt, self.iter_opts[opt]) @@ -1487,11 +1493,15 @@ def handle_iterate_opts(self): # pop first element from all *_list options as next value to use for opt in self.iter_opts: - if len(self.iter_opts[opt]) > self.iter_idx: - self.cfg[opt] = self.iter_opts[opt][self.iter_idx] - else: - self.cfg[opt] = '' # empty list => empty option as next value - self.log.debug("Next value for %s: %s" % (opt, str(self.cfg[opt]))) + if opt in ITERATE_OPTIONS: + if opt.startswith('iterate_'): + commonopt = opt[len('iterate_'):] + self.cfg[commonopt] = self.iter_opts[opt][self.iter_idx] + self.iter_opts[commonopt] + elif len(self.iter_opts[opt]) > self.iter_idx: + self.cfg[opt] = self.iter_opts[opt][self.iter_idx] + else: + self.cfg[opt] = '' # empty list => empty option as next value + self.log.debug("Next value for %s: %s" % (opt, str(self.cfg[opt]))) # re-enable templating before self.cfg values are used self.cfg.enable_templating = True @@ -1822,6 +1832,7 @@ def prepare_step(self, start_dir=True): # list of paths to include in RPATH filter; # only include builddir if we're not building in installation directory + self.rpath_filter_dirs = [] self.rpath_filter_dirs.append(tempfile.gettempdir()) if not self.build_in_installdir: self.rpath_filter_dirs.append(self.builddir) @@ -1829,6 +1840,7 @@ def prepare_step(self, start_dir=True): # always include '/lib', '/lib64', $ORIGIN, $ORIGIN/../lib and $ORIGIN/../lib64 # $ORIGIN will be resolved by the loader to be the full path to the executable or shared object # see also https://linux.die.net/man/8/ld-linux; + self.rpath_include_dirs = [] self.rpath_include_dirs.append(os.path.join(self.installdir, 'lib')) self.rpath_include_dirs.append(os.path.join(self.installdir, 'lib64')) self.rpath_include_dirs.append('$ORIGIN') @@ -2678,14 +2690,6 @@ def source_step_spec(initial): """Return source step specified.""" return get_step(SOURCE_STEP, "unpacking", source_substeps, True, initial=initial) - def prepare_step_spec(initial): - """Return prepare step specification.""" - if initial: - substeps = [lambda x: x.prepare_step] - else: - substeps = [lambda x: x.guess_start_dir] - return (PREPARE_STEP, 'preparing', substeps, False) - install_substeps = [ (False, lambda x: x.stage_install_step), (False, lambda x: x.make_installdir), @@ -2696,10 +2700,11 @@ def install_step_spec(initial): """Return install step specification.""" return get_step(INSTALL_STEP, "installing", install_substeps, True, initial=initial) - # format for step specifications: (stop_name: (description, list of functions, skippable)) + # format for step specifications: (step_name, description, list of functions, skippable) # core steps that are part of the iterated loop patch_step_spec = (PATCH_STEP, 'patching', [lambda x: x.patch_step], True) + prepare_step_spec = (PREPARE_STEP, 'preparing', [lambda x: x.prepare_step], False) configure_step_spec = (CONFIGURE_STEP, 'configuring', [lambda x: x.configure_step], True) build_step_spec = (BUILD_STEP, 'building', [lambda x: x.build_step], True) test_step_spec = (TEST_STEP, 'testing', [lambda x: x.test_step], True) @@ -2710,7 +2715,7 @@ def install_step_spec(initial): ready_step_spec(True), source_step_spec(True), patch_step_spec, - prepare_step_spec(True), + prepare_step_spec, configure_step_spec, build_step_spec, test_step_spec, @@ -2723,7 +2728,7 @@ def install_step_spec(initial): ready_step_spec(False), source_step_spec(False), patch_step_spec, - prepare_step_spec(False), + prepare_step_spec, configure_step_spec, build_step_spec, test_step_spec, diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index e4db6c48e4..9ba39ce760 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -142,6 +142,7 @@ # DEPENDENCIES easyconfig parameters 'allow_system_deps': [[], "Allow listed system dependencies (format: (, ))", DEPENDENCIES], 'builddependencies': [[], "List of build dependencies", DEPENDENCIES], + 'iterate_builddependencies': [[[]], "Iterated list of list of build dependencies", DEPENDENCIES], 'dependencies': [[], "List of dependencies", DEPENDENCIES], 'hiddendependencies': [[], "List of dependencies available as hidden modules", DEPENDENCIES], 'osdependencies': [[], "OS dependencies that should be present on the system", DEPENDENCIES], diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index dd9bd17324..618094a399 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -83,7 +83,8 @@ MANDATORY_PARAMS = ['name', 'version', 'homepage', 'description', 'toolchain'] # set of configure/build/install options that can be provided as lists for an iterated build -ITERATE_OPTIONS = ['preconfigopts', 'configopts', 'prebuildopts', 'buildopts', 'preinstallopts', 'installopts'] +ITERATE_OPTIONS = ['iterate_builddependencies', + 'preconfigopts', 'configopts', 'prebuildopts', 'buildopts', 'preinstallopts', 'installopts'] # name of easyconfigs archive subdirectory EASYCONFIGS_ARCHIVE_DIR = '__archive__' @@ -542,7 +543,12 @@ def parse(self): # parse dependency specifications # it's important that templating is still disabled at this stage! self.log.info("Parsing dependency specifications...") - self['builddependencies'] = [self._parse_dependency(dep, build_only=True) for dep in self['builddependencies']] + builddeps = [self._parse_dependency(dep, build_only=True) for dep in self['builddependencies']] + iterbuilddeps = [[self._parse_dependency(dep, build_only=True) for dep in x] + for x in self['iterate_builddependencies']] + self['iterate_builddependencies'] = iterbuilddeps + # temporary include all iter builddeps for finalize_dependencies + self['builddependencies'] = builddeps + [dep for x in iterbuilddeps for dep in x] self['dependencies'] = [self._parse_dependency(dep) for dep in self['dependencies']] self['hiddendependencies'] = [self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies']] @@ -554,6 +560,8 @@ def parse(self): # finalize dependencies w.r.t. minimal toolchains & module names self._finalize_dependencies() + # reset builddependencies to original value + self['builddependencies'] = builddeps # indicate that this is a parsed easyconfig self._config['parsed'] = [True, "This is a parsed easyconfig", "HIDDEN"] @@ -665,7 +673,7 @@ def validate_iterate_opts_lists(self): raise EasyBuildError("%s not available in self.cfg (anymore)?!", opt) # keep track of list, supply first element as first option to handle - if isinstance(self[opt], (list, tuple)): + if isinstance(self[opt], (list, tuple)) or opt.endswith('_iteropt'): opt_counts.append((opt, len(self[opt]))) # make sure that options that specify lists have the same length diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index 8157af8fed..078e3ea45f 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -63,7 +63,7 @@ ['homepage', 'description'], ['toolchain', 'toolchainopts'], ['source_urls', 'sources', 'patches', 'checksums'], - DEPENDENCY_PARAMETERS, + ['iterate_builddependencies'] + DEPENDENCY_PARAMETERS, ['osdependencies'], ['preconfigopts', 'configopts'], ['prebuildopts', 'buildopts'], diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index 21483101e1..9eb6b8f4bb 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -50,7 +50,7 @@ EB_FORMAT_EXTENSION = '.eb' # dependency parameters always need to be reformatted, to correctly deal with dumping parsed dependencies -REFORMAT_FORCED_PARAMS = ['sanity_check_paths'] + DEPENDENCY_PARAMETERS +REFORMAT_FORCED_PARAMS = ['sanity_check_paths', 'iterate_builddependencies'] + DEPENDENCY_PARAMETERS REFORMAT_SKIPPED_PARAMS = ['toolchain', 'toolchainopts'] REFORMAT_THRESHOLD_LENGTH = 100 # only reformat lines that would be longer than this amount of characters REFORMAT_ORDERED_ITEM_KEYS = { @@ -170,9 +170,15 @@ def _reformat_line(self, param_name, param_val, outer=False, addlen=0): for item in param_val: comment = self._get_item_comments(param_name, item).get(str(item), '') addlen = addlen + len(INDENT_4SPACES) + len(comment) + if isinstance(item, (list)): + itemstr = '[' + (",\n " + INDENT_4SPACES).join([ + self._reformat_line(param_name, subitem, outer=True, addlen=addlen) + for subitem in item]) + ']' + else: + itemstr = self._reformat_line(param_name, item, addlen=addlen) res += item_tmpl % { 'comment': comment, - 'item': self._reformat_line(param_name, item, addlen=addlen) + 'item': itemstr } # end with closing character: ], ), } @@ -236,6 +242,8 @@ def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_ # dependency easyconfig parameters were parsed, so these need special care to 'unparse' them if key in DEPENDENCY_PARAMETERS: valstr = [dump_dependency(d, ecfg['toolchain']) for d in val] + elif key == 'iterate_builddependencies': + valstr = [[dump_dependency(d, ecfg['toolchain']) for d in dep] for dep in val] elif key == 'toolchain': valstr = "{'name': '%(name)s', 'version': '%(version)s'}" % ecfg[key] else: diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb index fe6e2840f5..d8235ea0b4 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb @@ -5,19 +5,26 @@ versionsuffix = '-iter' homepage = 'https://easybuilders.github.io/easybuild' description = "Toy C program, 100% toy." -toolchain = {'name': 'dummy', 'version': 'dummy'} +toolchain = {'name': 'dummy', 'version': ''} sources = [SOURCE_TAR_GZ] patches = ['toy-0.0_fix-silly-typo-in-printf-statement.patch'] +iterate_builddependencies = [ + [('bzip2', '.1.0.6'), + ('GCC', '6.4.0-2.28')], + [('GCC', '6.4.0-2.28')], + [('GCC', '7.3.0-2.30')], +] + buildopts = [ '', - "-O2; mv %(name)s toy_O2", - "-O1; mv %(name)s toy_O1", + "-O2; mv %(name)s toy_O2_$EBVERSIONGCC", + "-O1; mv %(name)s toy_O1_$EBVERSIONGCC", ] sanity_check_paths = { - 'files': [('bin/yot', 'bin/toy'), 'bin/toy_O1', 'bin/toy_O2'], + 'files': [('bin/yot', 'bin/toy'), 'bin/toy_O1_7.3.0-2.30', 'bin/toy_O2_6.4.0-2.28'], 'dirs': ['bin'], } diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index ff1ec1c7a2..5bf8bb23d1 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1796,7 +1796,7 @@ def test_toy_iter(self): topdir = os.path.abspath(os.path.dirname(__file__)) toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-iter.eb') - expected_buildopts = ['', '-O2; mv %(name)s toy_O2', '-O1; mv %(name)s toy_O1'] + expected_buildopts = ['', '-O2; mv %(name)s toy_O2_$EBVERSIONGCC', '-O1; mv %(name)s toy_O1_$EBVERSIONGCC'] for extra_args in [None, ['--minimal-toolchains']]: # sanity check will make sure all entries in buildopts list were taken into account From e723d4bd2ede6d518ac01fa3fd765aabea576d56 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 30 Jan 2019 21:03:15 +0000 Subject: [PATCH 02/28] Make docstring raw to make Hound-ci happy. --- easybuild/framework/easyconfig/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 618094a399..2141027c5f 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -175,7 +175,7 @@ def det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, @toolchain_hierarchy_cache def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False): - """ + r""" Determine list of subtoolchains for specified parent toolchain. Result starts with the most minimal subtoolchains first, ends with specified toolchain. From fc0b990d2cd17ee67e578877d3d633abca0c7edf Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 31 Jan 2019 01:32:04 +0000 Subject: [PATCH 03/28] Fix dump of exts_list by special casing iterate_builddependencies. iterate_builddependencies is a list of lists of tuples unlike exts_list which is just a list of lists. --- easybuild/framework/easyconfig/format/one.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index 9eb6b8f4bb..da402cc797 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -52,6 +52,7 @@ # dependency parameters always need to be reformatted, to correctly deal with dumping parsed dependencies REFORMAT_FORCED_PARAMS = ['sanity_check_paths', 'iterate_builddependencies'] + DEPENDENCY_PARAMETERS REFORMAT_SKIPPED_PARAMS = ['toolchain', 'toolchainopts'] +REFORMAT_LIST_OF_LISTS_OF_TUPLES = ['iterate_builddependencies'] REFORMAT_THRESHOLD_LENGTH = 100 # only reformat lines that would be longer than this amount of characters REFORMAT_ORDERED_ITEM_KEYS = { 'sanity_check_paths': ['files', 'dirs'], @@ -140,6 +141,7 @@ def _reformat_line(self, param_name, param_val, outer=False, addlen=0): # note: this does not take into account the parameter name + '=', only the value line_too_long = len(param_strval) + addlen > REFORMAT_THRESHOLD_LENGTH forced = param_name in REFORMAT_FORCED_PARAMS + list_of_lists_of_tuples = param_name in REFORMAT_LIST_OF_LISTS_OF_TUPLES if param_name in REFORMAT_SKIPPED_PARAMS: self.log.info("Skipping reformatting value for parameter '%s'", param_name) @@ -170,7 +172,7 @@ def _reformat_line(self, param_name, param_val, outer=False, addlen=0): for item in param_val: comment = self._get_item_comments(param_name, item).get(str(item), '') addlen = addlen + len(INDENT_4SPACES) + len(comment) - if isinstance(item, (list)): + if isinstance(item, list) and list_of_lists_of_tuples: itemstr = '[' + (",\n " + INDENT_4SPACES).join([ self._reformat_line(param_name, subitem, outer=True, addlen=addlen) for subitem in item]) + ']' From 26259c795df5dfc9bc7def19bf72ce565d61fd6a Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 31 Jan 2019 02:11:49 +0000 Subject: [PATCH 04/28] Modify toy-0.0-iter dep to have both easyconfig and module available. --- test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb index d8235ea0b4..581d2e6602 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb @@ -11,8 +11,8 @@ sources = [SOURCE_TAR_GZ] patches = ['toy-0.0_fix-silly-typo-in-printf-statement.patch'] iterate_builddependencies = [ - [('bzip2', '.1.0.6'), - ('GCC', '6.4.0-2.28')], + [('GCC', '6.4.0-2.28'), + ('OpenMPI', '2.1.2', '', ('GCC', '6.4.0-2.28'))], [('GCC', '6.4.0-2.28')], [('GCC', '7.3.0-2.30')], ] From 66ce0c76761adc845e799ddef098ef2386cc51b9 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 31 Jan 2019 03:31:29 +0000 Subject: [PATCH 05/28] Restore initial environment when resetting the environment. --- easybuild/framework/easyblock.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 0fa13a0de4..346d6cb537 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -869,6 +869,13 @@ def make_builddir(self): trace_msg("build dir: %s" % self.builddir) + def reset_changes(self): + """ + Reset environment + """ + env.reset_changes() + restore_env(self.initial_environ) + def gen_installdir(self): """ Generate the name of the installation directory. @@ -2672,7 +2679,7 @@ def get_step(tag, descr, substeps, skippable, initial=True): ready_substeps = [ (False, lambda x: x.check_readiness_step), (True, lambda x: x.make_builddir), - (True, lambda x: env.reset_changes), + (True, lambda x: x.reset_changes), (True, lambda x: x.handle_iterate_opts), ] From 8c91557f0a562c7d8337da83c30d323aa4ad4f9b Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 31 Jan 2019 04:06:28 +0000 Subject: [PATCH 06/28] In reset environment for iterate reset to the env first encountered here. --- easybuild/framework/easyblock.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 346d6cb537..f0702e6b82 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -219,6 +219,7 @@ def __init__(self, ec): # keep track of initial environment we start in, so we can restore it if needed self.initial_environ = copy.deepcopy(os.environ) + self.reset_environ = None self.tweaked_env_vars = {} # should we keep quiet? @@ -874,7 +875,10 @@ def reset_changes(self): Reset environment """ env.reset_changes() - restore_env(self.initial_environ) + if self.reset_environ is None: + self.reset_environ = copy.deepcopy(os.environ) + else: + restore_env(self.reset_environ) def gen_installdir(self): """ From 8dd69f3a186a3901e957bd1d6a740715a20cc457 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 31 Jan 2019 12:46:33 +0000 Subject: [PATCH 07/28] Avoid temporary modification of builddeps for _finalize_dependencies() --- easybuild/framework/easyconfig/easyconfig.py | 22 ++++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 2141027c5f..963fbf8e22 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -543,12 +543,9 @@ def parse(self): # parse dependency specifications # it's important that templating is still disabled at this stage! self.log.info("Parsing dependency specifications...") - builddeps = [self._parse_dependency(dep, build_only=True) for dep in self['builddependencies']] - iterbuilddeps = [[self._parse_dependency(dep, build_only=True) for dep in x] - for x in self['iterate_builddependencies']] - self['iterate_builddependencies'] = iterbuilddeps - # temporary include all iter builddeps for finalize_dependencies - self['builddependencies'] = builddeps + [dep for x in iterbuilddeps for dep in x] + self['builddependencies'] = [self._parse_dependency(dep, build_only=True) for dep in self['builddependencies']] + self['iterate_builddependencies'] = [[self._parse_dependency(dep, build_only=True) for dep in x] + for x in self['iterate_builddependencies']] self['dependencies'] = [self._parse_dependency(dep) for dep in self['dependencies']] self['hiddendependencies'] = [self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies']] @@ -560,8 +557,6 @@ def parse(self): # finalize dependencies w.r.t. minimal toolchains & module names self._finalize_dependencies() - # reset builddependencies to original value - self['builddependencies'] = builddeps # indicate that this is a parsed easyconfig self._config['parsed'] = [True, "This is a parsed easyconfig", "HIDDEN"] @@ -1101,13 +1096,18 @@ def _finalize_dependencies(self): filter_deps = build_option('filter_deps') - for key in DEPENDENCY_PARAMETERS: + for key in DEPENDENCY_PARAMETERS + ['iterate_builddependencies']: # loop over a *copy* of dependency dicts (with resolved templates); # to update the original dep dict, we need to index with idx into self._config[key][0]... - for idx, dep in enumerate(self[key]): + val = self[key] + orig_val = self._config[key][0] + if key == 'iterate_builddependencies': + val = [dep for deps in val for dep in deps] + orig_val = [dep for deps in orig_val for dep in deps] + for idx, dep in enumerate(val): # reference to original dep dict, this is the one we should be updating - orig_dep = self._config[key][0][idx] + orig_dep = orig_val[idx] if filter_deps and orig_dep['name'] in filter_deps: self.log.debug("Skipping filtered dependency %s when finalising dependencies", orig_dep['name']) From 5c3beb2d6e9af1c9233b7e606eaffcfca01aebfa Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 31 Jan 2019 12:59:54 +0000 Subject: [PATCH 08/28] Merge iterate_builddependencies into DEPENDENCY_PARAMETERS. This way we make sure they are always considered. --- easybuild/framework/easyconfig/easyconfig.py | 2 +- easybuild/framework/easyconfig/format/format.py | 4 ++-- easybuild/framework/easyconfig/format/one.py | 8 ++++---- easybuild/framework/easyconfig/tweak.py | 9 +++++++-- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 963fbf8e22..4cbd731360 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1096,7 +1096,7 @@ def _finalize_dependencies(self): filter_deps = build_option('filter_deps') - for key in DEPENDENCY_PARAMETERS + ['iterate_builddependencies']: + for key in DEPENDENCY_PARAMETERS: # loop over a *copy* of dependency dicts (with resolved templates); # to update the original dep dict, we need to index with idx into self._config[key][0]... val = self[key] diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index 078e3ea45f..981a8d1be7 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -50,7 +50,7 @@ FORMAT_VERSION_REGEXP = re.compile(r'^#\s+%s\s*(?P\d+)\.(?P\d+)\s*$' % FORMAT_VERSION_KEYWORD, re.M) FORMAT_DEFAULT_VERSION = EasyVersion('1.0') -DEPENDENCY_PARAMETERS = ['builddependencies', 'dependencies', 'hiddendependencies'] +DEPENDENCY_PARAMETERS = ['iterate_builddependencies', 'builddependencies', 'dependencies', 'hiddendependencies'] # values for these keys will not be templated in dump() EXCLUDED_KEYS_REPLACE_TEMPLATES = ['description', 'easyblock', 'exts_list', 'homepage', 'name', 'toolchain', @@ -63,7 +63,7 @@ ['homepage', 'description'], ['toolchain', 'toolchainopts'], ['source_urls', 'sources', 'patches', 'checksums'], - ['iterate_builddependencies'] + DEPENDENCY_PARAMETERS, + DEPENDENCY_PARAMETERS, ['osdependencies'], ['preconfigopts', 'configopts'], ['prebuildopts', 'buildopts'], diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index da402cc797..d9aec04faf 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -50,7 +50,7 @@ EB_FORMAT_EXTENSION = '.eb' # dependency parameters always need to be reformatted, to correctly deal with dumping parsed dependencies -REFORMAT_FORCED_PARAMS = ['sanity_check_paths', 'iterate_builddependencies'] + DEPENDENCY_PARAMETERS +REFORMAT_FORCED_PARAMS = ['sanity_check_paths'] + DEPENDENCY_PARAMETERS REFORMAT_SKIPPED_PARAMS = ['toolchain', 'toolchainopts'] REFORMAT_LIST_OF_LISTS_OF_TUPLES = ['iterate_builddependencies'] REFORMAT_THRESHOLD_LENGTH = 100 # only reformat lines that would be longer than this amount of characters @@ -242,10 +242,10 @@ def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_ if val != default_values[key]: # dependency easyconfig parameters were parsed, so these need special care to 'unparse' them - if key in DEPENDENCY_PARAMETERS: - valstr = [dump_dependency(d, ecfg['toolchain']) for d in val] - elif key == 'iterate_builddependencies': + if key == 'iterate_builddependencies': valstr = [[dump_dependency(d, ecfg['toolchain']) for d in dep] for dep in val] + elif key in DEPENDENCY_PARAMETERS: + valstr = [dump_dependency(d, ecfg['toolchain']) for d in val] elif key == 'toolchain': valstr = "{'name': '%(name)s', 'version': '%(version)s'}" % ecfg[key] else: diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 627811d121..db524f4005 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -832,9 +832,14 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= for key in DEPENDENCY_PARAMETERS: # loop over a *copy* of dependency dicts (with resolved templates); # to update the original dep dict, we need to index with idx into self._config[key][0]... - for idx, dep in enumerate(parsed_ec['ec'][key]): + val = parced_ec['ec'][key] + orig_val = parced_ec['ec']._config[key][0] + if key == 'iterate_builddependencies': + val = [dep for deps in val for dep in deps] + orig_val = [dep for deps in orig_val for dep in deps] + for idx, dep in enumerate(val): # reference to original dep dict, this is the one we should be updating - orig_dep = parsed_ec['ec']._config[key][0][idx] + orig_dep = orig_val[idx] # skip dependencies that are marked as external modules if dep['external_module']: continue From a22dfb23382d529a4ee2a2b87dbac93df4a84a10 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 31 Jan 2019 13:08:25 +0000 Subject: [PATCH 09/28] Fix typo in parced. --- easybuild/framework/easyconfig/tweak.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index db524f4005..fad4dd76e1 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -832,8 +832,8 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= for key in DEPENDENCY_PARAMETERS: # loop over a *copy* of dependency dicts (with resolved templates); # to update the original dep dict, we need to index with idx into self._config[key][0]... - val = parced_ec['ec'][key] - orig_val = parced_ec['ec']._config[key][0] + val = parsed_ec['ec'][key] + orig_val = parsed_ec['ec']._config[key][0] if key == 'iterate_builddependencies': val = [dep for deps in val for dep in deps] orig_val = [dep for deps in orig_val for dep in deps] From f97624592e0f38d15a01e4f6a35e501919797f49 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 31 Jan 2019 13:18:36 +0000 Subject: [PATCH 10/28] Fix check for iterate_* keys in validate_iterate_opts_lists --- easybuild/framework/easyconfig/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 4cbd731360..a3249ab270 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -668,7 +668,7 @@ def validate_iterate_opts_lists(self): raise EasyBuildError("%s not available in self.cfg (anymore)?!", opt) # keep track of list, supply first element as first option to handle - if isinstance(self[opt], (list, tuple)) or opt.endswith('_iteropt'): + if isinstance(self[opt], (list, tuple)) or opt.startswith('iterate_'): opt_counts.append((opt, len(self[opt]))) # make sure that options that specify lists have the same length From f074e5174df0b9f85fd1c33f64f2838074b5fec1 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 31 Jan 2019 17:22:04 +0000 Subject: [PATCH 11/28] Keep iterate_builddependencies completely separate from builddependencies. --- easybuild/framework/easyblock.py | 20 ++++++-------------- easybuild/framework/easyconfig/easyconfig.py | 13 ++++++++----- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f0702e6b82..a9b5aaf599 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1491,11 +1491,7 @@ def handle_iterate_opts(self): # this will only be done during first iteration, since after that the options won't be lists anymore for opt in ITERATE_OPTIONS: # keep track of list, supply first element as first option to handle - if isinstance(self.cfg[opt], (list, tuple)) and self.cfg[opt] != [[]]: - if opt.startswith('iterate_') and self.iter_idx == 0: - # save original common value (e.g. builddependencies) - commonopt = opt[len('iterate_'):] - self.iter_opts[commonopt] = self.cfg[commonopt] + if isinstance(self.cfg[opt], (list, tuple)) and self.cfg[opt] and not isinstance(self.cfg[opt][0], dict): self.iter_opts[opt] = self.cfg[opt] # copy self.log.debug("Found list for %s: %s", opt, self.iter_opts[opt]) @@ -1504,15 +1500,11 @@ def handle_iterate_opts(self): # pop first element from all *_list options as next value to use for opt in self.iter_opts: - if opt in ITERATE_OPTIONS: - if opt.startswith('iterate_'): - commonopt = opt[len('iterate_'):] - self.cfg[commonopt] = self.iter_opts[opt][self.iter_idx] + self.iter_opts[commonopt] - elif len(self.iter_opts[opt]) > self.iter_idx: - self.cfg[opt] = self.iter_opts[opt][self.iter_idx] - else: - self.cfg[opt] = '' # empty list => empty option as next value - self.log.debug("Next value for %s: %s" % (opt, str(self.cfg[opt]))) + if len(self.iter_opts[opt]) > self.iter_idx: + self.cfg[opt] = self.iter_opts[opt][self.iter_idx] + else: + self.cfg[opt] = '' # empty list => empty option as next value + self.log.debug("Next value for %s: %s" % (opt, str(self.cfg[opt]))) # re-enable templating before self.cfg values are used self.cfg.enable_templating = True diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index a3249ab270..a6beff77ad 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -812,10 +812,9 @@ def dependencies(self, build_only=False): :param build_only: only return build dependencies, discard others """ - if build_only: - deps = self['builddependencies'] - else: - deps = self['dependencies'] + self['builddependencies'] + self['hiddendependencies'] + deps = self.builddependencies() + if not build_only: + deps = self['dependencies'] + deps + self['hiddendependencies'] # if filter-deps option is provided we "clean" the list of dependencies for # each processed easyconfig to remove the unwanted dependencies @@ -830,7 +829,11 @@ def builddependencies(self): """ return the parsed build dependencies """ - return self['builddependencies'] + iterbuilddeps = self['iterate_builddependencies'] or [] + if iterbuilddeps and not isinstance(iterbuilddeps[0], dict): + #flatten if not iterating yet + iterbuilddeps = [dep for deps in iterbuilddeps for dep in deps] + return iterbuilddeps + self['builddependencies'] @property def name(self): From f5def9e5470ec4632f8ce39ebb9eb3530eaab5aa Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 31 Jan 2019 17:28:07 +0000 Subject: [PATCH 12/28] Use flatten function instead of list comprehension to flatten. --- easybuild/framework/easyconfig/easyconfig.py | 8 ++++---- easybuild/framework/easyconfig/tweak.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index a6beff77ad..8479cf83c3 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -73,7 +73,7 @@ from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME, DUMMY_TOOLCHAIN_VERSION from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES, TOOLCHAIN_CAPABILITY_CUDA from easybuild.tools.toolchain.utilities import get_toolchain, search_toolchain -from easybuild.tools.utilities import quote_py_str, remove_unwanted_chars +from easybuild.tools.utilities import flatten, quote_py_str, remove_unwanted_chars from easybuild.tools.version import VERSION from easybuild.toolchains.compiler.cuda import Cuda @@ -832,7 +832,7 @@ def builddependencies(self): iterbuilddeps = self['iterate_builddependencies'] or [] if iterbuilddeps and not isinstance(iterbuilddeps[0], dict): #flatten if not iterating yet - iterbuilddeps = [dep for deps in iterbuilddeps for dep in deps] + iterbuilddeps = flatten(iterbuilddeps) return iterbuilddeps + self['builddependencies'] @property @@ -1105,8 +1105,8 @@ def _finalize_dependencies(self): val = self[key] orig_val = self._config[key][0] if key == 'iterate_builddependencies': - val = [dep for deps in val for dep in deps] - orig_val = [dep for deps in orig_val for dep in deps] + val = flatten(val) + orig_val = flatten(orig_val) for idx, dep in enumerate(val): # reference to original dep dict, this is the one we should be updating diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index fad4dd76e1..ee59295a5e 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -56,7 +56,7 @@ from easybuild.tools.robot import resolve_dependencies, robot_find_easyconfig from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES -from easybuild.tools.utilities import quote_str +from easybuild.tools.utilities import flatten, quote_str _log = fancylogger.getLogger('easyconfig.tweak', fname=False) @@ -835,8 +835,8 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= val = parsed_ec['ec'][key] orig_val = parsed_ec['ec']._config[key][0] if key == 'iterate_builddependencies': - val = [dep for deps in val for dep in deps] - orig_val = [dep for deps in orig_val for dep in deps] + val = flatten(val) + orig_val = flatten(orig_val) for idx, dep in enumerate(val): # reference to original dep dict, this is the one we should be updating orig_dep = orig_val[idx] From 3bf587368890d2e5ada7d77293419b9954e73a32 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 31 Jan 2019 17:58:46 +0000 Subject: [PATCH 13/28] Properly unload modules for iterate instead of messing with the environment. --- easybuild/framework/easyblock.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a9b5aaf599..a374332348 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -219,7 +219,6 @@ def __init__(self, ec): # keep track of initial environment we start in, so we can restore it if needed self.initial_environ = copy.deepcopy(os.environ) - self.reset_environ = None self.tweaked_env_vars = {} # should we keep quiet? @@ -870,16 +869,6 @@ def make_builddir(self): trace_msg("build dir: %s" % self.builddir) - def reset_changes(self): - """ - Reset environment - """ - env.reset_changes() - if self.reset_environ is None: - self.reset_environ = copy.deepcopy(os.environ) - else: - restore_env(self.reset_environ) - def gen_installdir(self): """ Generate the name of the installation directory. @@ -1850,9 +1839,17 @@ def prepare_step(self, start_dir=True): self.rpath_include_dirs.append('$ORIGIN/../lib') self.rpath_include_dirs.append('$ORIGIN/../lib64') + # in case of iterating builddependencies, unload any already loaded modules in reverse order + if 'iterate_builddependencies' in self.iter_opts: + if self.iter_idx > 0: + self.modules_tool.unload(reversed(self.loaded_modules)) + # do not unload modules that were loaded before the toolchain preparation. + orig_modules = self.modules_tool.loaded_modules() # prepare toolchain: load toolchain module and dependencies, set up build environment self.toolchain.prepare(self.cfg['onlytcmod'], deps=self.cfg.dependencies(), silent=self.silent, rpath_filter_dirs=self.rpath_filter_dirs, rpath_include_dirs=self.rpath_include_dirs) + if 'iterate_builddependencies' in self.iter_opts: + self.loaded_modules = self.modules_tool.loaded_modules()[len(orig_modules):] # keep track of environment variables that were tweaked and need to be restored after environment got reset # $TMPDIR may be tweaked for OpenMPI 2.x, which doesn't like long $TMPDIR paths... @@ -2675,7 +2672,7 @@ def get_step(tag, descr, substeps, skippable, initial=True): ready_substeps = [ (False, lambda x: x.check_readiness_step), (True, lambda x: x.make_builddir), - (True, lambda x: x.reset_changes), + (True, lambda x: env.reset_changes), (True, lambda x: x.handle_iterate_opts), ] From 641421b3f5279509bcee38fcaed59bbab4e47534 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 31 Jan 2019 18:01:50 +0000 Subject: [PATCH 14/28] Make Hound happy about comment. --- easybuild/framework/easyconfig/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 8479cf83c3..539fc2d1d2 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -831,7 +831,7 @@ def builddependencies(self): """ iterbuilddeps = self['iterate_builddependencies'] or [] if iterbuilddeps and not isinstance(iterbuilddeps[0], dict): - #flatten if not iterating yet + # flatten if not iterating yet iterbuilddeps = flatten(iterbuilddeps) return iterbuilddeps + self['builddependencies'] From ed6f73f4cc655ca8e9680d13283b3f73b28b7398 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 14 Feb 2019 19:43:30 +0000 Subject: [PATCH 15/28] Iterate builddependencies instead of iterate_builddependencies. --- easybuild/framework/easyblock.py | 8 +-- easybuild/framework/easyconfig/default.py | 1 - easybuild/framework/easyconfig/easyconfig.py | 59 ++++++++++++------- .../framework/easyconfig/format/format.py | 2 +- easybuild/framework/easyconfig/format/one.py | 11 ++-- easybuild/framework/easyconfig/tweak.py | 2 +- .../test_ecs/t/toy/toy-0.0-iter.eb | 2 +- 7 files changed, 50 insertions(+), 35 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a374332348..e3b1c774f4 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1478,9 +1478,9 @@ def handle_iterate_opts(self): # handle configure/build/install options that are specified as lists # set first element to be used, keep track of list in self.iter_opts # this will only be done during first iteration, since after that the options won't be lists anymore - for opt in ITERATE_OPTIONS: + if self.iter_idx == 0: # keep track of list, supply first element as first option to handle - if isinstance(self.cfg[opt], (list, tuple)) and self.cfg[opt] and not isinstance(self.cfg[opt][0], dict): + for opt in self.cfg.iterate_options: self.iter_opts[opt] = self.cfg[opt] # copy self.log.debug("Found list for %s: %s", opt, self.iter_opts[opt]) @@ -1840,7 +1840,7 @@ def prepare_step(self, start_dir=True): self.rpath_include_dirs.append('$ORIGIN/../lib64') # in case of iterating builddependencies, unload any already loaded modules in reverse order - if 'iterate_builddependencies' in self.iter_opts: + if 'builddependencies' in self.iter_opts: if self.iter_idx > 0: self.modules_tool.unload(reversed(self.loaded_modules)) # do not unload modules that were loaded before the toolchain preparation. @@ -1848,7 +1848,7 @@ def prepare_step(self, start_dir=True): # prepare toolchain: load toolchain module and dependencies, set up build environment self.toolchain.prepare(self.cfg['onlytcmod'], deps=self.cfg.dependencies(), silent=self.silent, rpath_filter_dirs=self.rpath_filter_dirs, rpath_include_dirs=self.rpath_include_dirs) - if 'iterate_builddependencies' in self.iter_opts: + if 'builddependencies' in self.iter_opts: self.loaded_modules = self.modules_tool.loaded_modules()[len(orig_modules):] # keep track of environment variables that were tweaked and need to be restored after environment got reset diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 9ba39ce760..e4db6c48e4 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -142,7 +142,6 @@ # DEPENDENCIES easyconfig parameters 'allow_system_deps': [[], "Allow listed system dependencies (format: (, ))", DEPENDENCIES], 'builddependencies': [[], "List of build dependencies", DEPENDENCIES], - 'iterate_builddependencies': [[[]], "Iterated list of list of build dependencies", DEPENDENCIES], 'dependencies': [[], "List of dependencies", DEPENDENCIES], 'hiddendependencies': [[], "List of dependencies available as hidden modules", DEPENDENCIES], 'osdependencies': [[], "OS dependencies that should be present on the system", DEPENDENCIES], diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index a493231594..a8d1aec081 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -83,7 +83,7 @@ MANDATORY_PARAMS = ['name', 'version', 'homepage', 'description', 'toolchain'] # set of configure/build/install options that can be provided as lists for an iterated build -ITERATE_OPTIONS = ['iterate_builddependencies', +ITERATE_OPTIONS = ['builddependencies', 'preconfigopts', 'configopts', 'prebuildopts', 'buildopts', 'preinstallopts', 'installopts'] # name of easyconfigs archive subdirectory @@ -540,12 +540,19 @@ def parse(self): run_hook(PARSE, hooks, args=[self], msg=parse_hook_msg) + # create a list of all options that are actually going to be iterated over + self.iterate_options = [opt for opt in ITERATE_OPTIONS + if (isinstance(self[opt], (list, tuple)) and + (opt != 'builddependencies' or (self[opt] and isinstance(self[opt][0][0], (list,tuple)))))] + # parse dependency specifications # it's important that templating is still disabled at this stage! self.log.info("Parsing dependency specifications...") - self['builddependencies'] = [self._parse_dependency(dep, build_only=True) for dep in self['builddependencies']] - self['iterate_builddependencies'] = [[self._parse_dependency(dep, build_only=True) for dep in x] - for x in self['iterate_builddependencies']] + if 'builddependencies' in self.iterate_options: + self['builddependencies'] = [[self._parse_dependency(dep, build_only=True) for dep in x] + for x in self['builddependencies']] + else: + self['builddependencies'] = [self._parse_dependency(dep, build_only=True) for dep in self['builddependencies']] self['dependencies'] = [self._parse_dependency(dep) for dep in self['dependencies']] self['hiddendependencies'] = [self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies']] @@ -668,7 +675,7 @@ def validate_iterate_opts_lists(self): raise EasyBuildError("%s not available in self.cfg (anymore)?!", opt) # keep track of list, supply first element as first option to handle - if isinstance(self[opt], (list, tuple)) or opt.startswith('iterate_'): + if opt in self.iterate_options: opt_counts.append((opt, len(self[opt]))) # make sure that options that specify lists have the same length @@ -684,29 +691,37 @@ def filter_hidden_deps(self): """ faulty_deps = [] + # templating must be temporarily disabled to obtain reference to original lists, + # so their elements can be changed in place + # see comments in resolve_template + enable_templating = self.enable_templating + self.enable_templating = False + orig_deps = dict([(key, self[key]) for key in ['dependencies', 'builddependencies', 'hiddendependencies']]) + self.enable_templating = enable_templating + + if 'builddependencies' in self.iterate_options: + deplists = self['builddependencies'] + [self['dependencies']] + orig_deplists = orig_deps['builddependencies'] + [orig_deps['dependencies']] + else: + deplists = [self['builddependencies'], self['dependencies']] + orig_deplists = [orig_deps['builddependencies'], orig_deps['dependencies']] + for hidden_idx, hidden_dep in enumerate(self['hiddendependencies']): hidden_mod_name = ActiveMNS().det_full_module_name(hidden_dep) visible_mod_name = ActiveMNS().det_full_module_name(hidden_dep, force_visible=True) # replace (build) dependencies with their equivalent hidden (build) dependency (if any) replaced = False - for key in ['builddependencies', 'dependencies']: - for idx, dep in enumerate(self[key]): + for deplist, orig_deplist in zip(deplists, orig_deplists): + for idx, dep in enumerate(deplist): dep_mod_name = dep['full_mod_name'] if dep_mod_name in [visible_mod_name, hidden_mod_name]: - # templating must be temporarily disabled to obtain reference to original lists, - # so their elements can be changed in place - # see comments in resolve_template - enable_templating = self.enable_templating - self.enable_templating = False # track whether this hidden dep is listed as a build dep - orig_hidden_dep = self['hiddendependencies'][hidden_idx] - if key == 'builddependencies': - orig_hidden_dep['build_only'] = True + orig_hidden_dep = orig_deps['hiddendependencies'][hidden_idx] + orig_hidden_dep['build_only'] = dep['build_only'] # actual replacement - self[key][idx] = orig_hidden_dep - self.enable_templating = enable_templating + orig_deplist[idx] = orig_hidden_dep replaced = True if dep_mod_name == visible_mod_name: @@ -837,11 +852,11 @@ def builddependencies(self): """ return the parsed build dependencies """ - iterbuilddeps = self['iterate_builddependencies'] or [] - if iterbuilddeps and not isinstance(iterbuilddeps[0], dict): + builddeps = self['builddependencies'] + if 'builddependencies' in self.iterate_options and not isinstance(builddeps[0], dict): # flatten if not iterating yet - iterbuilddeps = flatten(iterbuilddeps) - return iterbuilddeps + self['builddependencies'] + builddeps = flatten(builddeps) + return builddeps @property def name(self): @@ -1112,7 +1127,7 @@ def _finalize_dependencies(self): # to update the original dep dict, we need to index with idx into self._config[key][0]... val = self[key] orig_val = self._config[key][0] - if key == 'iterate_builddependencies': + if key in self.iterate_options: val = flatten(val) orig_val = flatten(orig_val) for idx, dep in enumerate(val): diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index 981a8d1be7..8157af8fed 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -50,7 +50,7 @@ FORMAT_VERSION_REGEXP = re.compile(r'^#\s+%s\s*(?P\d+)\.(?P\d+)\s*$' % FORMAT_VERSION_KEYWORD, re.M) FORMAT_DEFAULT_VERSION = EasyVersion('1.0') -DEPENDENCY_PARAMETERS = ['iterate_builddependencies', 'builddependencies', 'dependencies', 'hiddendependencies'] +DEPENDENCY_PARAMETERS = ['builddependencies', 'dependencies', 'hiddendependencies'] # values for these keys will not be templated in dump() EXCLUDED_KEYS_REPLACE_TEMPLATES = ['description', 'easyblock', 'exts_list', 'homepage', 'name', 'toolchain', diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index 093d51d3b2..1d561a298a 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -52,7 +52,7 @@ # dependency parameters always need to be reformatted, to correctly deal with dumping parsed dependencies REFORMAT_FORCED_PARAMS = ['sanity_check_paths'] + DEPENDENCY_PARAMETERS REFORMAT_SKIPPED_PARAMS = ['toolchain', 'toolchainopts'] -REFORMAT_LIST_OF_LISTS_OF_TUPLES = ['iterate_builddependencies'] +REFORMAT_LIST_OF_LISTS_OF_TUPLES = ['builddependencies'] REFORMAT_THRESHOLD_LENGTH = 100 # only reformat lines that would be longer than this amount of characters REFORMAT_ORDERED_ITEM_KEYS = { 'sanity_check_paths': ['files', 'dirs'], @@ -236,10 +236,11 @@ def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_ val = ecfg[key] if val != default_values[key]: # dependency easyconfig parameters were parsed, so these need special care to 'unparse' them - if key == 'iterate_builddependencies': - valstr = [[dump_dependency(d, ecfg['toolchain']) for d in dep] for dep in val] - elif key in DEPENDENCY_PARAMETERS: - valstr = [dump_dependency(d, ecfg['toolchain']) for d in val] + if key in DEPENDENCY_PARAMETERS: + if key in ecfg.iterate_options: + valstr = [[dump_dependency(d, ecfg['toolchain']) for d in dep] for dep in val] + else: + valstr = [dump_dependency(d, ecfg['toolchain']) for d in val] elif key == 'toolchain': valstr = "{'name': '%(name)s', 'version': '%(version)s'}" % ecfg[key] else: diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index ee59295a5e..8442140dcc 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -834,7 +834,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= # to update the original dep dict, we need to index with idx into self._config[key][0]... val = parsed_ec['ec'][key] orig_val = parsed_ec['ec']._config[key][0] - if key == 'iterate_builddependencies': + if key in parsed_ec['ec'].iterate_options: val = flatten(val) orig_val = flatten(orig_val) for idx, dep in enumerate(val): diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb index 581d2e6602..b680ad8b27 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb @@ -10,7 +10,7 @@ toolchain = {'name': 'dummy', 'version': ''} sources = [SOURCE_TAR_GZ] patches = ['toy-0.0_fix-silly-typo-in-printf-statement.patch'] -iterate_builddependencies = [ +builddependencies = [ [('GCC', '6.4.0-2.28'), ('OpenMPI', '2.1.2', '', ('GCC', '6.4.0-2.28'))], [('GCC', '6.4.0-2.28')], From 8d6f3bbf56e195e165e8443f6a073d873b1e3b57 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 14 Feb 2019 20:09:08 +0000 Subject: [PATCH 16/28] Hound-ci changes. --- easybuild/framework/easyconfig/easyconfig.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index a8d1aec081..f44573fe82 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -542,17 +542,19 @@ def parse(self): # create a list of all options that are actually going to be iterated over self.iterate_options = [opt for opt in ITERATE_OPTIONS - if (isinstance(self[opt], (list, tuple)) and - (opt != 'builddependencies' or (self[opt] and isinstance(self[opt][0][0], (list,tuple)))))] + if (isinstance(self[opt], (list, tuple)) and + (opt != 'builddependencies' or + (self[opt] and isinstance(self[opt][0][0], (list, tuple)))))] # parse dependency specifications # it's important that templating is still disabled at this stage! self.log.info("Parsing dependency specifications...") if 'builddependencies' in self.iterate_options: - self['builddependencies'] = [[self._parse_dependency(dep, build_only=True) for dep in x] - for x in self['builddependencies']] + self['builddependencies'] = [[self._parse_dependency(dep, build_only=True) + for dep in x] for x in self['builddependencies']] else: - self['builddependencies'] = [self._parse_dependency(dep, build_only=True) for dep in self['builddependencies']] + self['builddependencies'] = [self._parse_dependency(dep, build_only=True) + for dep in self['builddependencies']] self['dependencies'] = [self._parse_dependency(dep) for dep in self['dependencies']] self['hiddendependencies'] = [self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies']] From b51ad11e9445a3a9b03ee3cfd8d645aea9243425 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Fri, 15 Feb 2019 13:10:01 +0000 Subject: [PATCH 17/28] yaml easyconfigs builddeps already a list of dicts, fixing that. --- easybuild/framework/easyconfig/easyconfig.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index f44573fe82..e1f6522678 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -541,20 +541,20 @@ def parse(self): run_hook(PARSE, hooks, args=[self], msg=parse_hook_msg) # create a list of all options that are actually going to be iterated over + # builddependencies are always a list, need to look deeper down below self.iterate_options = [opt for opt in ITERATE_OPTIONS - if (isinstance(self[opt], (list, tuple)) and - (opt != 'builddependencies' or - (self[opt] and isinstance(self[opt][0][0], (list, tuple)))))] + if opt != 'builddependencies' and isinstance(self[opt], (list, tuple))] # parse dependency specifications # it's important that templating is still disabled at this stage! self.log.info("Parsing dependency specifications...") - if 'builddependencies' in self.iterate_options: - self['builddependencies'] = [[self._parse_dependency(dep, build_only=True) - for dep in x] for x in self['builddependencies']] + builddeps = self['builddependencies'] + if builddeps and isinstance(builddeps[0], (list, tuple)) and isinstance(builddeps[0][0], (list, tuple)): + self.iterate_options.append('builddependencies') + builddeps = [[self._parse_dependency(dep, build_only=True) for dep in x] for x in builddeps] else: - self['builddependencies'] = [self._parse_dependency(dep, build_only=True) - for dep in self['builddependencies']] + builddeps = [self._parse_dependency(dep, build_only=True) for dep in builddeps] + self['builddependencies'] = builddeps self['dependencies'] = [self._parse_dependency(dep) for dep in self['dependencies']] self['hiddendependencies'] = [self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies']] From fcdc90bac0b19456f3fab9e987f8e4909dd70965 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Feb 2019 11:20:55 +0100 Subject: [PATCH 18/28] add & use EasyConfig.get_ref method + code cleanup & minor style fixes --- easybuild/framework/easyblock.py | 47 +++++------ easybuild/framework/easyconfig/easyconfig.py | 80 ++++++++++++------- easybuild/framework/easyconfig/format/one.py | 12 +-- easybuild/framework/easyconfig/tweak.py | 23 +++--- test/framework/easyconfig.py | 32 ++++++++ .../test_ecs/t/toy/toy-0.0-iter.eb | 2 +- 6 files changed, 124 insertions(+), 72 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index e3b1c774f4..08d89e3c0b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -457,9 +457,7 @@ def fetch_extension_sources(self, skip_checksums=False): Find source file for extensions. """ exts_sources = [] - self.cfg.enable_templating = False - exts_list = self.cfg['exts_list'] - self.cfg.enable_templating = True + exts_list = self.cfg.get_ref('exts_list') if self.dry_run: self.dry_run_msg("\nList of sources/patches for extensions:") @@ -477,9 +475,7 @@ def fetch_extension_sources(self, skip_checksums=False): # make sure we grab *raw* dict of default options for extension, # since it may use template values like %(name)s & %(version)s - self.cfg.enable_templating = False - ext_options = copy.deepcopy(self.cfg['exts_default_options']) - self.cfg.enable_templating = True + ext_options = copy.deepcopy(self.cfg.get_ref('exts_default_options')) def_src_tmpl = "%(name)s-%(version)s.tar.gz" @@ -1373,10 +1369,8 @@ def skip_extensions(self): - use this to detect existing extensions and to remove them from self.exts - based on initial R version """ - # disabling templating is required here to support legacy string templates like name/version - self.cfg.enable_templating = False - exts_filter = self.cfg['exts_filter'] - self.cfg.enable_templating = True + # obtaining untemplated reference value is required here to support legacy string templates like name/version + exts_filter = self.cfg.get_ref('exts_filter') if not exts_filter or len(exts_filter) == 0: raise EasyBuildError("Skipping of extensions, but no exts_filter set in easyconfig") @@ -1473,11 +1467,12 @@ def handle_iterate_opts(self): """Handle options relevant during iterated part of build/install procedure.""" # disable templating in this function, since we're messing about with values in self.cfg + prev_enable_templating = self.cfg.enable_templating self.cfg.enable_templating = False - # handle configure/build/install options that are specified as lists + # handle configure/build/install options that are specified as lists (+ perhaps builddependencies) # set first element to be used, keep track of list in self.iter_opts - # this will only be done during first iteration, since after that the options won't be lists anymore + # only needs to be done during first iteration, since after that the options won't be lists anymore if self.iter_idx == 0: # keep track of list, supply first element as first option to handle for opt in self.cfg.iterate_options: @@ -1487,7 +1482,7 @@ def handle_iterate_opts(self): if self.iter_opts: self.log.info("Current iteration index: %s", self.iter_idx) - # pop first element from all *_list options as next value to use + # pop first element from all iterative easyconfig parameters as next value to use for opt in self.iter_opts: if len(self.iter_opts[opt]) > self.iter_idx: self.cfg[opt] = self.iter_opts[opt][self.iter_idx] @@ -1496,7 +1491,7 @@ def handle_iterate_opts(self): self.log.debug("Next value for %s: %s" % (opt, str(self.cfg[opt]))) # re-enable templating before self.cfg values are used - self.cfg.enable_templating = True + self.cfg.enable_templating = prev_enable_templating # prepare for next iteration (if any) self.iter_idx += 1 @@ -1504,6 +1499,7 @@ def handle_iterate_opts(self): def restore_iterate_opts(self): """Restore options that were iterated over""" # disable templating, since we're messing about with values in self.cfg + prev_enable_templating = self.cfg.enable_templating self.cfg.enable_templating = False for opt in self.iter_opts: @@ -1511,7 +1507,7 @@ def restore_iterate_opts(self): self.log.debug("Restored value of '%s' that was iterated over: %s", opt, self.cfg[opt]) # re-enable templating before self.cfg values are used - self.cfg.enable_templating = True + self.cfg.enable_templating = prev_enable_templating def det_iter_cnt(self): """Determine iteration count based on configure/build/install options that may be lists.""" @@ -1824,20 +1820,20 @@ def prepare_step(self, start_dir=True): # list of paths to include in RPATH filter; # only include builddir if we're not building in installation directory - self.rpath_filter_dirs = [] - self.rpath_filter_dirs.append(tempfile.gettempdir()) + self.rpath_filter_dirs = [tempfile.gettempdir()] if not self.build_in_installdir: self.rpath_filter_dirs.append(self.builddir) # always include '/lib', '/lib64', $ORIGIN, $ORIGIN/../lib and $ORIGIN/../lib64 # $ORIGIN will be resolved by the loader to be the full path to the executable or shared object # see also https://linux.die.net/man/8/ld-linux; - self.rpath_include_dirs = [] - self.rpath_include_dirs.append(os.path.join(self.installdir, 'lib')) - self.rpath_include_dirs.append(os.path.join(self.installdir, 'lib64')) - self.rpath_include_dirs.append('$ORIGIN') - self.rpath_include_dirs.append('$ORIGIN/../lib') - self.rpath_include_dirs.append('$ORIGIN/../lib64') + self.rpath_include_dirs = [ + os.path.join(self.installdir, 'lib'), + os.path.join(self.installdir, 'lib64'), + '$ORIGIN', + '$ORIGIN/../lib', + '$ORIGIN/../lib64', + ] # in case of iterating builddependencies, unload any already loaded modules in reverse order if 'builddependencies' in self.iter_opts: @@ -1853,6 +1849,7 @@ def prepare_step(self, start_dir=True): # keep track of environment variables that were tweaked and need to be restored after environment got reset # $TMPDIR may be tweaked for OpenMPI 2.x, which doesn't like long $TMPDIR paths... + self.tweaked_env_vars = {} for var in ['TMPDIR']: if os.environ.get(var) != self.initial_environ.get(var): self.tweaked_env_vars[var] = os.environ.get(var) @@ -3275,9 +3272,7 @@ def make_checksum_lines(checksums, indent_level): # make sure we grab *raw* dict of default options for extension, # since it may use template values like %(name)s & %(version)s - app.cfg.enable_templating = False - exts_default_options = app.cfg['exts_default_options'] - app.cfg.enable_templating = True + exts_default_options = app.cfg.get_ref('exts_default_options') for key, val in sorted(ext_options.items()): if key != 'checksums' and val != exts_default_options.get(key): diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index e1f6522678..ec98b2bd10 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -548,15 +548,18 @@ def parse(self): # parse dependency specifications # it's important that templating is still disabled at this stage! self.log.info("Parsing dependency specifications...") + self['dependencies'] = [self._parse_dependency(dep) for dep in self['dependencies']] + self['hiddendependencies'] = [self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies']] + + # need to take into account that builddependencies may need to be iterated over, + # i.e. when the value is a list of lists of tuples builddeps = self['builddependencies'] - if builddeps and isinstance(builddeps[0], (list, tuple)) and isinstance(builddeps[0][0], (list, tuple)): + if builddeps and all(isinstance(x, (list, tuple)) for b in builddeps for x in b): self.iterate_options.append('builddependencies') builddeps = [[self._parse_dependency(dep, build_only=True) for dep in x] for x in builddeps] else: builddeps = [self._parse_dependency(dep, build_only=True) for dep in builddeps] self['builddependencies'] = builddeps - self['dependencies'] = [self._parse_dependency(dep) for dep in self['dependencies']] - self['hiddendependencies'] = [self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies']] # restore templating self.enable_templating = prev_enable_templating @@ -670,15 +673,14 @@ def validate_iterate_opts_lists(self): # when lists are used, they should be all of same length # list of length 1 are treated as if it were strings in EasyBlock opt_counts = [] - for opt in ITERATE_OPTIONS: + for opt in self.iterate_options: # anticipate changes in available easyconfig parameters (e.g. makeopts -> buildopts?) if self.get(opt, None) is None: raise EasyBuildError("%s not available in self.cfg (anymore)?!", opt) # keep track of list, supply first element as first option to handle - if opt in self.iterate_options: - opt_counts.append((opt, len(self[opt]))) + opt_counts.append((opt, len(self[opt]))) # make sure that options that specify lists have the same length list_opt_lengths = [length for (opt, length) in opt_counts if length > 1] @@ -693,37 +695,33 @@ def filter_hidden_deps(self): """ faulty_deps = [] - # templating must be temporarily disabled to obtain reference to original lists, - # so their elements can be changed in place - # see comments in resolve_template - enable_templating = self.enable_templating - self.enable_templating = False - orig_deps = dict([(key, self[key]) for key in ['dependencies', 'builddependencies', 'hiddendependencies']]) - self.enable_templating = enable_templating + # obtain reference to original lists, so their elements can be changed in place + deps = dict([(key, self.get_ref(key)) for key in ['dependencies', 'builddependencies', 'hiddendependencies']]) if 'builddependencies' in self.iterate_options: - deplists = self['builddependencies'] + [self['dependencies']] - orig_deplists = orig_deps['builddependencies'] + [orig_deps['dependencies']] + deplists = deps['builddependencies'] else: - deplists = [self['builddependencies'], self['dependencies']] - orig_deplists = [orig_deps['builddependencies'], orig_deps['dependencies']] + deplists = [deps['builddependencies']] - for hidden_idx, hidden_dep in enumerate(self['hiddendependencies']): + deplists.append(deps['dependencies']) + + for hidden_idx, hidden_dep in enumerate(deps['hiddendependencies']): hidden_mod_name = ActiveMNS().det_full_module_name(hidden_dep) visible_mod_name = ActiveMNS().det_full_module_name(hidden_dep, force_visible=True) # replace (build) dependencies with their equivalent hidden (build) dependency (if any) replaced = False - for deplist, orig_deplist in zip(deplists, orig_deplists): + for deplist in deplists: for idx, dep in enumerate(deplist): dep_mod_name = dep['full_mod_name'] if dep_mod_name in [visible_mod_name, hidden_mod_name]: # track whether this hidden dep is listed as a build dep - orig_hidden_dep = orig_deps['hiddendependencies'][hidden_idx] - orig_hidden_dep['build_only'] = dep['build_only'] + hidden_dep = deps['hiddendependencies'][hidden_idx] + hidden_dep['build_only'] = dep['build_only'] + # actual replacement - orig_deplist[idx] = orig_hidden_dep + deplist[idx] = hidden_dep replaced = True if dep_mod_name == visible_mod_name: @@ -855,7 +853,7 @@ def builddependencies(self): return the parsed build dependencies """ builddeps = self['builddependencies'] - if 'builddependencies' in self.iterate_options and not isinstance(builddeps[0], dict): + if 'builddependencies' in self.iterate_options and builddeps and not isinstance(builddeps[0], dict): # flatten if not iterating yet builddeps = flatten(builddeps) return builddeps @@ -1126,16 +1124,20 @@ def _finalize_dependencies(self): for key in DEPENDENCY_PARAMETERS: # loop over a *copy* of dependency dicts (with resolved templates); - # to update the original dep dict, we need to index with idx into self._config[key][0]... - val = self[key] - orig_val = self._config[key][0] + deps = self[key] + + # to update the original dep dict, we need to get a reference with templating disabled... + deps_ref = self.get_ref(key) + + # take into account that this *dependencies parameters may be iterated over if key in self.iterate_options: - val = flatten(val) - orig_val = flatten(orig_val) - for idx, dep in enumerate(val): + deps = flatten(deps) + deps_ref = flatten(deps_ref) + + for idx, dep in enumerate(deps): # reference to original dep dict, this is the one we should be updating - orig_dep = orig_val[idx] + orig_dep = deps_ref[idx] if filter_deps and orig_dep['name'] in filter_deps: self.log.debug("Skipping filtered dependency %s when finalising dependencies", orig_dep['name']) @@ -1242,6 +1244,24 @@ def __getitem__(self, key): return value + def get_ref(self, key): + """ + Obtain reference to original/untemplated value of specified easyconfig parameter + rather than a copied value with templated values. + """ + # see also comments in resolve_template + + # temporarily disable templating + prev_enable_templating = self.enable_templating + self.enable_templating = False + + ref = self[key] + + # restore previous value for 'enable_templating' + self.enable_templating = prev_enable_templating + + return ref + @handle_deprecated_or_replaced_easyconfig_parameters def __setitem__(self, key, value): """Set value of specified easyconfig parameter (help text & co is left untouched)""" diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index 1d561a298a..7af9528f82 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -30,7 +30,6 @@ :author: Stijn De Weirdt (Ghent University) :author: Kenneth Hoste (Ghent University) """ -import copy import os import re import tempfile @@ -141,7 +140,7 @@ def _reformat_line(self, param_name, param_val, outer=False, addlen=0): # note: this does not take into account the parameter name + '=', only the value line_too_long = len(param_strval) + addlen > REFORMAT_THRESHOLD_LENGTH forced = param_name in REFORMAT_FORCED_PARAMS - list_of_lists_of_tuples = param_name in REFORMAT_LIST_OF_LISTS_OF_TUPLES + list_of_lists_of_tuples_param = param_name in REFORMAT_LIST_OF_LISTS_OF_TUPLES if param_name in REFORMAT_SKIPPED_PARAMS: self.log.info("Skipping reformatting value for parameter '%s'", param_name) @@ -172,12 +171,14 @@ def _reformat_line(self, param_name, param_val, outer=False, addlen=0): for item in param_val: comment = self._get_item_comments(param_name, item).get(str(item), '') addlen = addlen + len(INDENT_4SPACES) + len(comment) - if isinstance(item, list) and list_of_lists_of_tuples: - itemstr = '[' + (",\n " + INDENT_4SPACES).join([ + is_list_of_lists_of_tuples = isinstance(item, list) and all(isinstance(x, tuple) for x in item) + if list_of_lists_of_tuples_param and is_list_of_lists_of_tuples: + itemstr = '[' + (',\n ' + INDENT_4SPACES).join([ self._reformat_line(param_name, subitem, outer=True, addlen=addlen) for subitem in item]) + ']' else: itemstr = self._reformat_line(param_name, item, addlen=addlen) + res += item_tmpl % { 'comment': comment, 'item': itemstr @@ -235,7 +236,8 @@ def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_ for key in group: val = ecfg[key] if val != default_values[key]: - # dependency easyconfig parameters were parsed, so these need special care to 'unparse' them + # dependency easyconfig parameters were parsed, so these need special care to 'unparse' them; + # take into account that these parameters may be iterative (i.e. a list of lists of parsed deps) if key in DEPENDENCY_PARAMETERS: if key in ecfg.iterate_options: valstr = [[dump_dependency(d, ecfg['toolchain']) for d in dep] for dep in val] diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 8442140dcc..b65b3fb43e 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -820,23 +820,26 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= :return: Location of the modified easyconfig file """ # Fully parse the original easyconfig - parsed_ec = process_easyconfig(ec_spec, validate=False)[0] + parsed_ec = process_easyconfig(ec_spec, validate=False)[0]['ec'] + # Replace the toolchain if the mapping exists - tc_name = parsed_ec['ec']['toolchain']['name'] + tc_name = parsed_ec['toolchain']['name'] if tc_name in toolchain_mapping: new_toolchain = toolchain_mapping[tc_name] - _log.debug("Replacing parent toolchain %s with %s", parsed_ec['ec']['toolchain'], new_toolchain) - parsed_ec['ec']['toolchain'] = new_toolchain + _log.debug("Replacing parent toolchain %s with %s", parsed_ec['toolchain'], new_toolchain) + parsed_ec['toolchain'] = new_toolchain # Replace the toolchains of all the dependencies for key in DEPENDENCY_PARAMETERS: # loop over a *copy* of dependency dicts (with resolved templates); - # to update the original dep dict, we need to index with idx into self._config[key][0]... - val = parsed_ec['ec'][key] - orig_val = parsed_ec['ec']._config[key][0] - if key in parsed_ec['ec'].iterate_options: + # to update the original dep dict, we need to get a reference with templating disabled... + val = parsed_ec[key] + orig_val = parsed_ec.get_ref(key) + + if key in parsed_ec.iterate_options: val = flatten(val) orig_val = flatten(orig_val) + for idx, dep in enumerate(val): # reference to original dep dict, this is the one we should be updating orig_dep = orig_val[idx] @@ -853,10 +856,10 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name(dep) orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(dep) # Determine the name of the modified easyconfig and dump it to target_dir - ec_filename = '%s-%s.eb' % (parsed_ec['ec']['name'], det_full_ec_version(parsed_ec['ec'])) + ec_filename = '%s-%s.eb' % (parsed_ec['name'], det_full_ec_version(parsed_ec)) tweaked_spec = os.path.join(targetdir or tempfile.gettempdir(), ec_filename) - parsed_ec['ec'].dump(tweaked_spec, always_overwrite=False, backup=True) + parsed_ec.dump(tweaked_spec, always_overwrite=False, backup=True) _log.debug("Dumped easyconfig tweaked via --try-toolchain* to %s", tweaked_spec) return tweaked_spec diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index b8ef923a01..b7bfd93189 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -2438,6 +2438,38 @@ def test_filename(self): ec = EasyConfig(test_ec) self.assertTrue(ec.filename(), os.path.basename(test_ec)) + def test_get_ref(self): + """Test get_ref method.""" + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-iter.eb')) + + # without using get_ref, we get a (templated) copy rather than the original value + sources = ec['sources'] + self.assertEqual(sources, ['toy-0.0.tar.gz']) + self.assertFalse(sources is ec._config['sources'][0]) + + # same for .get + sources = ec.get('sources') + self.assertEqual(sources, ['toy-0.0.tar.gz']) + self.assertFalse(sources is ec._config['sources'][0]) + + # with get_ref, we get the original untemplated value + sources_ref = ec.get_ref('sources') + self.assertEqual(sources_ref, ['%(name)s-%(version)s.tar.gz']) + self.assertTrue(sources_ref is ec._config['sources'][0]) + + sanity_check_paths_ref = ec.get_ref('sanity_check_paths') + self.assertTrue(sanity_check_paths_ref is ec._config['sanity_check_paths'][0]) + + # also items inside are still references to original (i.e. not copies) + self.assertTrue(sanity_check_paths_ref['files'] is ec._config['sanity_check_paths'][0]['files']) + + # get_ref also works for values other than lists/dicts + self.assertEqual(ec['description'], "Toy C program, 100% toy.") + descr_ref = ec.get_ref('description') + self.assertEqual(descr_ref, "Toy C program, 100% %(name)s.") + self.assertTrue(descr_ref is ec._config['description'][0]) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb index b680ad8b27..085768f707 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb @@ -3,7 +3,7 @@ version = '0.0' versionsuffix = '-iter' homepage = 'https://easybuilders.github.io/easybuild' -description = "Toy C program, 100% toy." +description = "Toy C program, 100% %(name)s." toolchain = {'name': 'dummy', 'version': ''} From 4ff572261d9f3b30dff8a88cd7d748fb4a49e451 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Fri, 1 Mar 2019 20:02:07 +0000 Subject: [PATCH 19/28] Restore environment when iterating instead of use modules. This reverts commit 3bf5873. --- easybuild/framework/easyblock.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 08d89e3c0b..4c24f058db 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -219,6 +219,7 @@ def __init__(self, ec): # keep track of initial environment we start in, so we can restore it if needed self.initial_environ = copy.deepcopy(os.environ) + self.reset_environ = None self.tweaked_env_vars = {} # should we keep quiet? @@ -865,6 +866,16 @@ def make_builddir(self): trace_msg("build dir: %s" % self.builddir) + def reset_changes(self): + """ + Reset environment + """ + env.reset_changes() + if self.reset_environ is None: + self.reset_environ = copy.deepcopy(os.environ) + else: + restore_env(self.reset_environ) + def gen_installdir(self): """ Generate the name of the installation directory. @@ -1835,17 +1846,9 @@ def prepare_step(self, start_dir=True): '$ORIGIN/../lib64', ] - # in case of iterating builddependencies, unload any already loaded modules in reverse order - if 'builddependencies' in self.iter_opts: - if self.iter_idx > 0: - self.modules_tool.unload(reversed(self.loaded_modules)) - # do not unload modules that were loaded before the toolchain preparation. - orig_modules = self.modules_tool.loaded_modules() # prepare toolchain: load toolchain module and dependencies, set up build environment self.toolchain.prepare(self.cfg['onlytcmod'], deps=self.cfg.dependencies(), silent=self.silent, rpath_filter_dirs=self.rpath_filter_dirs, rpath_include_dirs=self.rpath_include_dirs) - if 'builddependencies' in self.iter_opts: - self.loaded_modules = self.modules_tool.loaded_modules()[len(orig_modules):] # keep track of environment variables that were tweaked and need to be restored after environment got reset # $TMPDIR may be tweaked for OpenMPI 2.x, which doesn't like long $TMPDIR paths... @@ -2669,7 +2672,7 @@ def get_step(tag, descr, substeps, skippable, initial=True): ready_substeps = [ (False, lambda x: x.check_readiness_step), (True, lambda x: x.make_builddir), - (True, lambda x: env.reset_changes), + (True, lambda x: x.reset_changes), (True, lambda x: x.handle_iterate_opts), ] From aa29163ab615d3dbae1c0183a39662ff4a14a7e3 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 14 Mar 2019 16:15:24 -0400 Subject: [PATCH 20/28] Introduce "iterating" easyconfig field. This allows (build)dependencies() be flattened strictly outside the iterating process. --- easybuild/framework/easyblock.py | 6 ++++++ easybuild/framework/easyconfig/easyconfig.py | 9 ++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 4c24f058db..b8e3031d5e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1481,6 +1481,9 @@ def handle_iterate_opts(self): prev_enable_templating = self.cfg.enable_templating self.cfg.enable_templating = False + # tell easyconfig we are iterating (used by dependencies() and builddependencies()) + self.cfg.iterating = True + # handle configure/build/install options that are specified as lists (+ perhaps builddependencies) # set first element to be used, keep track of list in self.iter_opts # only needs to be done during first iteration, since after that the options won't be lists anymore @@ -1517,6 +1520,9 @@ def restore_iterate_opts(self): self.cfg[opt] = self.iter_opts[opt] self.log.debug("Restored value of '%s' that was iterated over: %s", opt, self.cfg[opt]) + # tell easyconfig we are no longer iterating (used by dependencies() and builddependencies()) + self.cfg.iterating = False + # re-enable templating before self.cfg values are used self.cfg.enable_templating = prev_enable_templating diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index ec98b2bd10..25f8de895d 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -544,6 +544,7 @@ def parse(self): # builddependencies are always a list, need to look deeper down below self.iterate_options = [opt for opt in ITERATE_OPTIONS if opt != 'builddependencies' and isinstance(self[opt], (list, tuple))] + self.iterating = False # parse dependency specifications # it's important that templating is still disabled at this stage! @@ -831,6 +832,7 @@ def dependencies(self, build_only=False): """ Returns an array of parsed dependencies (after filtering, if requested) dependency = {'name': '', 'version': '', 'dummy': (False|True), 'versionsuffix': '', 'toolchain': ''} + Iterable builddependencies are flattened when not iterating. :param build_only: only return build dependencies, discard others """ @@ -850,11 +852,12 @@ def dependencies(self, build_only=False): def builddependencies(self): """ - return the parsed build dependencies + Return a flat list of the parsed build dependencies + When builddependencies are iterable they are flattened outside + of the iterating process, because the callers want a simple list. """ builddeps = self['builddependencies'] - if 'builddependencies' in self.iterate_options and builddeps and not isinstance(builddeps[0], dict): - # flatten if not iterating yet + if 'builddependencies' in self.iterate_options and self.iterating: builddeps = flatten(builddeps) return builddeps From 72c48cdfa02747593fd126fa3d526703f58950f2 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Fri, 15 Mar 2019 03:02:31 +0000 Subject: [PATCH 21/28] Correct boolean check for iterating. --- easybuild/framework/easyconfig/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 25f8de895d..1aed8a8f7b 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -857,7 +857,7 @@ def builddependencies(self): of the iterating process, because the callers want a simple list. """ builddeps = self['builddependencies'] - if 'builddependencies' in self.iterate_options and self.iterating: + if 'builddependencies' in self.iterate_options and not self.iterating: builddeps = flatten(builddeps) return builddeps From e8b393b29b0bec05e4a6fef23b95f851cba35f07 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Fri, 15 Mar 2019 03:05:42 +0000 Subject: [PATCH 22/28] Remove duplicates in flattened lists of builddependencies. --- easybuild/framework/easyconfig/easyconfig.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 1aed8a8f7b..2fe6314496 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -853,12 +853,18 @@ def dependencies(self, build_only=False): def builddependencies(self): """ Return a flat list of the parsed build dependencies - When builddependencies are iterable they are flattened outside - of the iterating process, because the callers want a simple list. + When builddependencies are iterable they are flattened lists with + duplicates removed outside of the iterating process, because the callers + want simple lists. """ builddeps = self['builddependencies'] if 'builddependencies' in self.iterate_options and not self.iterating: - builddeps = flatten(builddeps) + tmp = flatten(builddeps) + # remove duplicates + builddeps = [] + for builddep in tmp: + if builddep not in builddeps: + builddeps.append(builddep) return builddeps @property From b0579ad0bf3c3579d7d9134500f861f2c8317616 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Fri, 15 Mar 2019 03:08:42 +0000 Subject: [PATCH 23/28] Fix typo in parameters spotted by @boegel. --- easybuild/framework/easyconfig/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 2fe6314496..6a7a2d1ae4 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1138,7 +1138,7 @@ def _finalize_dependencies(self): # to update the original dep dict, we need to get a reference with templating disabled... deps_ref = self.get_ref(key) - # take into account that this *dependencies parameters may be iterated over + # take into account that this *dependencies parameter may be iterated over if key in self.iterate_options: deps = flatten(deps) deps_ref = flatten(deps_ref) From 248dc25a0d5d75732e7011fcfaea047026f927e4 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Fri, 15 Mar 2019 03:11:49 +0000 Subject: [PATCH 24/28] Add comment about reset_environ. --- easybuild/framework/easyblock.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index b8e3031d5e..3cf3f9c329 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -868,7 +868,10 @@ def make_builddir(self): def reset_changes(self): """ - Reset environment + Reset environment. + When iterating over builddependencies, every time we start a new iteration + we need to restore the environment to where it was before the relevant modules + were loaded. """ env.reset_changes() if self.reset_environ is None: From bddaa0da568ae7c1288bbd0a0ca3ab9e60d4adb1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 15 Mar 2019 16:08:34 +0100 Subject: [PATCH 25/28] rename 'EasyBlock.reset_changes' to 'EasyBlock.reset_env' --- easybuild/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 3cf3f9c329..c4b503edc4 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -866,7 +866,7 @@ def make_builddir(self): trace_msg("build dir: %s" % self.builddir) - def reset_changes(self): + def reset_env(self): """ Reset environment. When iterating over builddependencies, every time we start a new iteration @@ -2681,7 +2681,7 @@ def get_step(tag, descr, substeps, skippable, initial=True): ready_substeps = [ (False, lambda x: x.check_readiness_step), (True, lambda x: x.make_builddir), - (True, lambda x: x.reset_changes), + (True, lambda x: x.reset_env), (True, lambda x: x.handle_iterate_opts), ] From 7c34a92899f6180ca910d6c507a0097829120ce8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 15 Mar 2019 16:15:26 +0100 Subject: [PATCH 26/28] use 'nub' to filter out duplicates from flattened list of build dependencies --- easybuild/framework/easyconfig/easyconfig.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 6a7a2d1ae4..9593722e38 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -858,13 +858,11 @@ def builddependencies(self): want simple lists. """ builddeps = self['builddependencies'] + if 'builddependencies' in self.iterate_options and not self.iterating: - tmp = flatten(builddeps) - # remove duplicates - builddeps = [] - for builddep in tmp: - if builddep not in builddeps: - builddeps.append(builddep) + # flatten and remove duplicates + builddeps = nub(flatten(builddeps)) + return builddeps @property From 96c253b87a417572a9e6968aba8a4242a753d211 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 15 Mar 2019 17:12:54 +0100 Subject: [PATCH 27/28] can't use nub to filter out duplicates in flattened list of build dependencies --- easybuild/framework/easyconfig/easyconfig.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 9593722e38..ef96d1a464 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -860,8 +860,11 @@ def builddependencies(self): builddeps = self['builddependencies'] if 'builddependencies' in self.iterate_options and not self.iterating: - # flatten and remove duplicates - builddeps = nub(flatten(builddeps)) + # flatten and remove duplicates (can't use 'nub', since dict values are not hashable) + builddeps = [] + for dep in flatten(builddeps): + if dep not in builddeps: + builddeps.append(dep) return builddeps From 49452751054c30e524ba13aea26f9665a3e6e3bb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 15 Mar 2019 17:49:28 +0100 Subject: [PATCH 28/28] fix filtering of duplicates in builddependencies method (@boegel should take a (Python) programming class...) --- easybuild/framework/easyconfig/easyconfig.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index ef96d1a464..8ebcbc1f18 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -861,8 +861,9 @@ def builddependencies(self): if 'builddependencies' in self.iterate_options and not self.iterating: # flatten and remove duplicates (can't use 'nub', since dict values are not hashable) + all_builddeps = flatten(builddeps) builddeps = [] - for dep in flatten(builddeps): + for dep in all_builddeps: if dep not in builddeps: builddeps.append(dep)