[formatv2] split off supported section from default section, add preliminary support for a dependencies section

easybuild/framework/easyconfig/format/two.py

@@ -108,8 +108,8 @@ def get_config_dict(self):
         version = self.specs.get('version', None)
         if version is None:
             # check for default version
-            if 'default_version' in cov.default:
-                version = cov.default['default_version']
+            if 'version' in cov.default:
+                version = cov.default['version']
                 self.log.info("no software version specified, using default version '%s'" % version)
             else:
                 self.log.error("no software version specified, no default version found")
@@ -120,13 +120,13 @@ def get_config_dict(self):
         toolchain_name = tc_spec.get('name', None)
         if toolchain_name is None:
             # check for default toolchain
-            if 'default_toolchain' in cov.default:
-                toolchain = cov.default['default_toolchain']
-                toolchain_name = toolchain.tc_name
+            if 'toolchain' in cov.default:
+                toolchain = cov.default['toolchain']
+                toolchain_name = toolchain['name']
                 self.log.info("no toolchain name specified, using default '%s'" % toolchain_name)
                 toolchain_version = tc_spec.get('version', None)
                 if toolchain_version is None:
-                    toolchain_version = toolchain.get_version_str()
+                    toolchain_version = toolchain['version']
                     self.log.info("no toolchain version specified, using default '%s'" % toolchain_version)
             else:
                 self.log.error("no toolchain name specified, no default toolchain found")
@@ -136,23 +136,6 @@ def get_config_dict(self):
             if toolchain_version is None:
                 self.log.error("Toolchain specification incomplete: name %s provided, but no version" % toolchain_name)
 
-        # add version/toolchain specifications to config dict
-        if isinstance(version, VersionOperator):
-            version = version.get_version_str()
-        elif not isinstance(version, basestring):
-            self.log.error("Found version of unexpected type: %s (%s)" % (type(version), version))
-        if isinstance(toolchain_version, ToolchainVersionOperator):
-            toolchain_version = toolchain_version.get_version_str()
-        elif not isinstance(toolchain_version, basestring):
-            tup = (type(toolchain_version), toolchain_version)
-            self.log.error("Found toolchain version of unexpected type: %s (%s)" % tup)
-
-        cfg.update({
-            'version': version,
-            'toolchain': {'name': toolchain_name, 'version': toolchain_version},
-        })
-        self.log.debug("Config dict including version/toolchain specs: %s" % cfg)
-
         # toolchain name is known, remove all others toolchains from parsed easyconfig before we continue
         # this also performs some validation, and checks for conflicts between section markers
         self.log.debug("sections for full parsed configobj: %s" % cov.sections)
@@ -161,7 +144,14 @@ def get_config_dict(self):
 
         section_specs = cov.get_specs_for(version=version, tcname=toolchain_name, tcversion=toolchain_version)
         cfg.update(section_specs)
+        self.log.debug("Config dict after processing applicable easyconfig sections: %s" % cfg)
         # FIXME what about updating dict values/appending to list values? how do we allow both redefining and updating? = and +=?
 
-        self.log.debug("Final config dict: %s" % cfg)
+        # update config with correct version/toolchain (to avoid using values specified in default section)
+        cfg.update({
+            'version': version,
+            'toolchain': {'name': toolchain_name, 'version': toolchain_version},
+        })
+
+        self.log.debug("Final config dict (including correct version/toolchain): %s" % cfg)
         return cfg

easybuild/framework/easyconfig/format/version.py

@@ -475,6 +475,17 @@ def parse_versop_str(self, tcversop_str):
         self.log.debug("toolchain versop expression '%s' parsed to '%s'" % (tcversop_str, tcversop_dict))
         return tcversop_dict
 
+    def as_dict(self):
+        """
+        Return toolchain version operator as a dictionary with name/version keys.
+        Returns None if translation to a dictionary is not possible (e.g. non-equals operator, missing version, ...).
+        """
+        version = self.get_version_str()
+        if not None in [self.tc_name, version] and self.operator == self.OPERATOR_MAP['==']:
+            return {'name': self.tc_name, 'version': version}
+        else:
+            return None
+
 
 class OrderedVersionOperators(object):
     """
@@ -566,9 +577,11 @@ class ConfigObjVersion(object):
     """
     # TODO: add nested/recursive example to docstring
 
-    DEFAULT = 'DEFAULT'
+    SECTION_MARKER_DEFAULT = 'DEFAULT'
+    SECTION_MARKER_DEPENDENCIES = 'DEPENDENCIES'
+    SECTION_MARKER_SUPPORTED = 'SUPPORTED'
     # list of known marker types (except default)
-    KNOWN_MARKER_TYPES = [ToolchainVersionOperator, VersionOperator]  # order matters, see parse_sections
+    KNOWN_VERSION_MARKER_TYPES = [ToolchainVersionOperator, VersionOperator]  # order matters, see parse_sections
     VERSION_OPERATOR_VALUE_TYPES = {
         # toolchains: comma-separated list of toolchain version operators
         'toolchains': ToolchainVersionOperator,
@@ -586,8 +599,9 @@ def __init__(self, configobj=None):
         self.tcname = None
 
         self.default = {}  # default section
-        self.sections = {}  # non-default sections
-        self.unfiltered_sections = {}  # unfiltered non-default sections
+        self.supported = {}  # supported section
+        self.sections = {}  # all other sections
+        self.unfiltered_sections = {}  # unfiltered other sections
 
         self.versops = OrderedVersionOperators()
         self.tcversops = OrderedVersionOperators()
@@ -621,13 +635,29 @@ def parse_sections(self, configobj, toparse=None, parent=None, depth=0):
         for key, value in toparse.items():
             if isinstance(value, Section):
                 self.log.debug("Enter subsection key %s value %s" % (key, value))
-                # only 3 types of sectionkeys supported: VersionOperator, ToolchainVersionOperator, and DEFAULT
-                if key in [self.DEFAULT]:
+                # only supported types of section keys are:
+                # * DEFAULT
+                # * SUPPORTED
+                # * dependencies
+                # * VersionOperator or ToolchainVersionOperator (e.g. [> 2.0], [goolf > 1])
+                if key in [self.SECTION_MARKER_DEFAULT, self.SECTION_MARKER_SUPPORTED]:
+                    # parse value as a section, recursively
                     new_key = key
+                    new_value = self.parse_sections(configobj, toparse=value, parent=value.parent, depth=value.depth)
+
+                elif key == self.SECTION_MARKER_DEPENDENCIES:
+                    new_key = 'dependencies'
+                    new_value = []
+                    for dep_name, dep_spec in value.items():
+                        if isinstance(dep_spec, Section):
+                            self.log.error("Unsupported nested section '%s' found in dependencies section" % dep_name)
+                        else:
+                            # FIXME: parse the dependency specification for version, toolchain, suffix, etc.
+                            new_value.append((dep_name, dep_spec))
                 else:
                     # try parsing key as toolchain version operator first
                     # try parsing as version operator if it's not a toolchain version operator
-                    for marker_type in self.KNOWN_MARKER_TYPES:
+                    for marker_type in self.KNOWN_VERSION_MARKER_TYPES:
                         new_key = marker_type(key)
                         if new_key:
                             self.log.debug("'%s' was parsed as a %s section marker" % (key, marker_type.__name__))
@@ -637,8 +667,8 @@ def parse_sections(self, configobj, toparse=None, parent=None, depth=0):
                     if not new_key:
                         self.log.error("Unsupported section marker '%s'" % key)
 
-                # parse value as a section, recursively
-                new_value = self.parse_sections(configobj, toparse=value, parent=value.parent, depth=value.depth)
+                    # parse value as a section, recursively
+                    new_value = self.parse_sections(configobj, toparse=value, parent=value.parent, depth=value.depth)
 
             else:
                 new_key = key
@@ -710,7 +740,7 @@ def validate_and_filter_by_toolchain(self, tcname, processed=None, filtered_sect
                     self.versops.add(key, value)
                     filtered_sections[key] = value
                 else:
-                    self.log.error("Unhandled section marker type '%s', not in %s?" % (type(key), self.KNOWN_MARKER_TYPES))
+                    self.log.error("Unhandled section marker '%s' (type '%s')" % (key, type(key)))
 
                 # recursively go deeper for (relevant) sections
                 self.validate_and_filter_by_toolchain(tcname, processed=value, filtered_sections=filtered_sections,
@@ -744,28 +774,35 @@ def parse(self, configobj):
         # process the configobj instance
         self.sections = self.parse_sections(self.configobj)
 
-        # check for defaults section
-        default = self.sections.pop(self.DEFAULT, {})
-        known_default_keywords = ('toolchains', 'versions')
-        # default should only have versions and toolchains
+        # handle default section
         # no nesting
-        #  - add DEFAULT key,values to the root of self.sections
+        #  - add DEFAULT key-value entries to the root of self.sections
         #  - key-value items from other sections will be deeper down
         #  - deepest level is best match and wins, so defaults are on top level
+        default = self.sections.pop(self.SECTION_MARKER_DEFAULT, {})
         for key, value in default.items():
-            if not key in known_default_keywords:
-                self.log.error('Unsupported key %s in %s section' % (key, self.DEFAULT))
             self.sections[key] = value
+        self.default = default
 
-        if 'versions' in default:
+        # handle supported section
+        # supported should only have 'versions' and 'toolchains' keys
+        supported = self.sections.pop(self.SECTION_MARKER_SUPPORTED, {})
+        known_supported_keywords = self.VERSION_OPERATOR_VALUE_TYPES.keys()
+        for key, value in supported.items():
+            if not key in known_supported_keywords:
+                self.log.error('Unsupported key %s in %s section' % (key, self.SECTION_MARKER_SUPPORTED))
+            self.sections['%s' % key] = value
+        self.supported = supported
+
+        if 'versions' in supported:
             # first of list is special: it is the default
-            default['default_version'] = default['versions'][0]
-        if 'toolchains' in default:
+            self.default['version'] = supported['versions'][0].get_version_str()
+        if 'toolchains' in supported:
             # first of list is special: it is the default
-            default['default_toolchain'] = default['toolchains'][0]
+            self.default['toolchain'] = supported['toolchains'][0].as_dict()
 
-        self.default = default
-        self.log.debug("(parse) default: %s; sections: %s" % (self.default, self.sections))
+        tup = (self.default, self.supported, self.sections)
+        self.log.debug("(parse) default: %s; supported: %s, sections: %s" % tup)
 
     def get_specs_for(self, version=None, tcname=None, tcversion=None):
         """
@@ -775,26 +812,22 @@ def get_specs_for(self, version=None, tcname=None, tcversion=None):
             cfg = self.default.dict()
         else:
             cfg = copy.deepcopy(self.default)
-        # get rid of entries that are not easyconfig parameters
-        # FIXME these should go in a dedicated section SUPPORTED instead of DEFAULT?
-        for key in ['default_toolchain', 'default_version', 'toolchains', 'versions']:
-            cfg.pop(key)
 
         # make sure that requested version/toolchain are supported by this easyconfig
-        versions = [x.get_version_str() for x in self.default['versions']]
+        versions = [x.get_version_str() for x in self.supported['versions']]
         if version is None:
             self.log.debug("No version specified")
         elif version in versions:
             self.log.debug("Version '%s' is supported in easyconfig." % version)
         else:
             self.log.error("Version '%s' not supported in easyconfig (only %s)" % (version, versions))
 
-        tcnames = [tc.tc_name for tc in self.default['toolchains']]
+        tcnames = [tc.tc_name for tc in self.supported['toolchains']]
         if tcname is None:
             self.log.debug("Toolchain name not specified.")
         elif tcname in tcnames:
             self.log.debug("Toolchain '%s' is supported in easyconfig." % tcname)
-            tcversions = [tc.get_version_str() for tc in self.default['toolchains'] if tc.tc_name == tcname]
+            tcversions = [tc.get_version_str() for tc in self.supported['toolchains'] if tc.tc_name == tcname]
             if tcversion is None:
                 self.log.debug("Toolchain version not specified.")
             elif tcversion in tcversions:

test/framework/easyconfigs/v2.0/GCC.eb

@@ -33,7 +33,7 @@ license = GPLv3
 
 moduleclass = 'compiler'
 
-[DEFAULT]
+[SUPPORTED]
 versions = 4.6.2, 4.6.3
 toolchains = dummy == dummy
 

test/framework/easyconfigs/v2.0/doesnotexist.eb

@@ -23,10 +23,12 @@ source_urls = ['http://ftpmirror.gnu.org/gzip']
 # make sure the gzip and gunzip binaries are available after installation
 sanity_check_paths = {'files': ["bin/gunzip", "bin/gzip"], 'dirs': []}
 
-[DEFAULT]
+[SUPPORTED]
 versions=1.0.0,>= 0.5.0
 toolchains=dummy > 5,GCC == 3.0.0,ictce < 1.0
+
 [goolf > 1]
 versions=1.5.0
+
 [2.0.0]
 toolchains=dummy > 6,GCC > 4

test/framework/easyconfigs/v2.0/gzip.eb

@@ -20,8 +20,9 @@ sanity_check_paths = {
 
 sanity_check_commands = [True, ('gzip', '--version')]
 
-moduleclass = 'tools'
-
-[DEFAULT]
+[SUPPORTED]
 versions = 1.4, 1.5
 toolchains = dummy == dummy, goolf, GCC == 4.6.3, goolf == 1.4.10, ictce == 4.1.13
+
+[DEFAULT]
+moduleclass = tools

test/framework/easyconfigs/v2.0/toy.eb

@@ -20,6 +20,6 @@ sanity_check_paths = {
 
 moduleclass = 'tools'
 
-[DEFAULT]
+[SUPPORTED]
 versions = 1.0, 0.0
 toolchains = goolf == 1.4.10, dummy == dummy