take into account custom configuration options specified in easystack file + drop support for easystack files using 'software' top-level key

easybuild/framework/easystack.py

@@ -28,11 +28,13 @@
 :author: Denis Kristak (Inuits)
 :author: Pavel Grochal (Inuits)
 :author: Kenneth Hoste (HPC-UGent)
+:author: Caspar van Leeuwen (SURF)
 """
+import pprint
+
 from easybuild.base import fancylogger
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.filetools import read_file
-from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
 from easybuild.tools.py2vs3 import string_type
 from easybuild.tools.utilities import only_if_module_is_available
 try:
@@ -69,30 +71,13 @@ class EasyStack(object):
     def __init__(self):
         self.easybuild_version = None
         self.robot = False
-        self.software_list = []
-        self.easyconfigs = []  # A list of easyconfig names. May or may not include .eb extension
-        # A dict where keys are easyconfig names, values are dictionary of options that should be
-        # applied for that easyconfig
-        self.ec_opts = {}
-
-    def compose_ec_filenames(self):
-        """Returns a list of all easyconfig names"""
-        ec_filenames = []
-
-        # entries specified via 'software' top-level key
-        for sw in self.software_list:
-            full_ec_version = det_full_ec_version({
-                'toolchain': {'name': sw.toolchain_name, 'version': sw.toolchain_version},
-                'version': sw.version,
-                'versionsuffix': sw.versionsuffix,
-            })
-            ec_filename = '%s-%s.eb' % (sw.name, full_ec_version)
-            ec_filenames.append(ec_filename)
-
-        # entries specified via 'easyconfigs' top-level key
-        for ec in self.easyconfigs:
-            ec_filenames.append(ec)
-        return ec_filenames
+        self.ec_opt_tuples = []  # A list of tuples (easyconfig_name, eaysconfig_specific_opts)
+
+    def __str__(self):
+        """
+        Pretty printing of an EasyStack instance
+        """
+        return pprint.pformat(self.ec_opt_tuples)
 
     # flags applicable to all sw (i.e. robot)
     def get_general_options(self):
@@ -129,37 +114,33 @@ def parse(filepath):
         except yaml.YAMLError as err:
             raise EasyBuildError("Failed to parse %s: %s" % (filepath, err))
 
-        easystack_data = None
-        top_keys = ('easyconfigs', 'software')
-        keys_found = []
-        for key in top_keys:
-            if key in easystack_raw:
-                keys_found.append(key)
-        # For now, we don't support mixing multiple top_keys, so check that only one was defined
-        if len(keys_found) > 1:
-            keys_string = ', '.join(keys_found)
-            msg = "Specifying multiple top level keys (%s) " % keys_string
-            msg += "in one EasyStack file is currently not supported"
-            msg += ", see %s for documentation." % EASYSTACK_DOC_URL
-            raise EasyBuildError(msg)
-        elif len(keys_found) == 0:
-            msg = "Not a valid EasyStack YAML file: no 'easyconfigs' or 'software' top-level key found"
-            msg += ", see %s for documentation." % EASYSTACK_DOC_URL
-            raise EasyBuildError(msg)
-        else:
-            key = keys_found[0]
+        key = 'easyconfigs'
+        if key in easystack_raw:
             easystack_data = easystack_raw[key]
-
-        parse_method_name = 'parse_by_' + key
-        parse_method = getattr(EasyStackParser, 'parse_by_%s' % key, None)
-        if parse_method is None:
-            raise EasyBuildError("Easystack parse method '%s' not found!", parse_method_name)
+            if isinstance(easystack_data, dict) or isinstance(easystack_data, str):
+                datatype = 'dict' if isinstance(easystack_data, dict) else 'str'
+                msg = '\n'.join([
+                    "Found %s value for '%s' in %s, should be list." % (datatype, key, filepath),
+                    "Make sure you use '-' to create list items under '%s', for example:" % key,
+                    "    easyconfigs:",
+                    "        - example-1.0.eb",
+                    "        - example-2.0.eb:",
+                    "            options:"
+                    "              ...",
+                ])
+                raise EasyBuildError(msg)
+            elif not isinstance(easystack_data, list):
+                raise EasyBuildError("Value type for '%s' in %s should be list, found %s",
+                                     key, filepath, type(easystack_data))
+        else:
+            raise EasyBuildError("Top-level key '%s' missing in easystack file %s", key, filepath)
 
         # assign general easystack attributes
         easybuild_version = easystack_raw.get('easybuild_version', None)
         robot = easystack_raw.get('robot', False)
 
-        return parse_method(filepath, easystack_data, easybuild_version=easybuild_version, robot=robot)
+        return EasyStackParser.parse_by_easyconfigs(filepath, easystack_data,
+                                                    easybuild_version=easybuild_version, robot=robot)
 
     @staticmethod
     def parse_by_easyconfigs(filepath, easyconfigs, easybuild_version=None, robot=False):
@@ -173,7 +154,7 @@ def parse_by_easyconfigs(filepath, easyconfigs, easybuild_version=None, robot=Fa
             if isinstance(easyconfig, str):
                 if not easyconfig.endswith('.eb'):
                     easyconfig = easyconfig + '.eb'
-                easystack.easyconfigs.append(easyconfig)
+                easystack.ec_opt_tuples.append((easyconfig, None))
             elif isinstance(easyconfig, dict):
                 if len(easyconfig) == 1:
                     # Get single key from dictionary 'easyconfig'
@@ -183,132 +164,25 @@ def parse_by_easyconfigs(filepath, easyconfigs, easybuild_version=None, robot=Fa
                         easyconf_name_with_eb = easyconf_name + '.eb'
                     else:
                         easyconf_name_with_eb = easyconf_name
-                    easystack.easyconfigs.append(easyconf_name_with_eb)
-                    # Add options to the ec_opts dict
-                    if 'options' in easyconfig[easyconf_name].keys():
-                        easystack.ec_opts[easyconf_name_with_eb] = easyconfig[easyconf_name]['options']
+                    # Get options
+                    ec_dict = easyconfig[easyconf_name] or {}
+
+                    # make sure only 'options' key is used (for now)
+                    if any(x != 'options' for x in ec_dict):
+                        msg = "Found one or more invalid keys for %s (only 'options' supported): %s"
+                        raise EasyBuildError(msg, easyconf_name, ', '.join(sorted(ec_dict.keys())))
+
+                    opts = ec_dict.get('options')
+                    easystack.ec_opt_tuples.append((easyconf_name_with_eb, opts))
                 else:
-                    dict_keys = ', '.join(easyconfig.keys())
+                    dict_keys = ', '.join(sorted(easyconfig.keys()))
                     msg = "Failed to parse easystack file: expected a dictionary with one key (the EasyConfig name), "
                     msg += "instead found keys: %s" % dict_keys
                     msg += ", see %s for documentation." % EASYSTACK_DOC_URL
                     raise EasyBuildError(msg)
 
         return easystack
 
-    @staticmethod
-    def parse_by_software(filepath, software, easybuild_version=None, robot=False):
-        """
-        Parse easystack file with 'software' as top-level key.
-        """
-
-        easystack = EasyStack()
-
-        # assign software-specific easystack attributes
-        for name in software:
-            # ensure we have a string value (YAML parser returns type = dict
-            # if levels under the current attribute are present)
-            check_value(name, "software name")
-            try:
-                toolchains = software[name]['toolchains']
-            except KeyError:
-                raise EasyBuildError("Toolchains for software '%s' are not defined in %s", name, filepath)
-            for toolchain in toolchains:
-                check_value(toolchain, "software %s" % name)
-
-                if toolchain == 'SYSTEM':
-                    toolchain_name, toolchain_version = 'system', ''
-                else:
-                    toolchain_parts = toolchain.split('-', 1)
-                    if len(toolchain_parts) == 2:
-                        toolchain_name, toolchain_version = toolchain_parts
-                    elif len(toolchain_parts) == 1:
-                        toolchain_name, toolchain_version = toolchain, ''
-                    else:
-                        raise EasyBuildError("Incorrect toolchain specification for '%s' in %s, too many parts: %s",
-                                             name, filepath, toolchain_parts)
-
-                try:
-                    # if version string containts asterisk or labels, raise error (asterisks not supported)
-                    versions = toolchains[toolchain]['versions']
-                except TypeError as err:
-                    wrong_structure_err = "An error occurred when interpreting "
-                    wrong_structure_err += "the data for software %s: %s" % (name, err)
-                    raise EasyBuildError(wrong_structure_err)
-                if '*' in str(versions):
-                    asterisk_err = "EasyStack specifications of '%s' in %s contain asterisk. "
-                    asterisk_err += "Wildcard feature is not supported yet."
-                    raise EasyBuildError(asterisk_err, name, filepath)
-
-                # yaml versions can be in different formats in yaml file
-                # firstly, check if versions in yaml file are read as a dictionary.
-                # Example of yaml structure:
-                # ========================================================================
-                # versions:
-                #   '2.25':
-                #   '2.23':
-                #     versionsuffix: '-R-4.0.0'
-                # ========================================================================
-                if isinstance(versions, dict):
-                    for version in versions:
-                        check_value(version, "%s (with %s toolchain)" % (name, toolchain_name))
-                        if versions[version] is not None:
-                            version_spec = versions[version]
-                            if 'versionsuffix' in version_spec:
-                                versionsuffix = str(version_spec['versionsuffix'])
-                            else:
-                                versionsuffix = ''
-                            if 'exclude-labels' in str(version_spec) or 'include-labels' in str(version_spec):
-                                lab_err = "EasyStack specifications of '%s' in %s "
-                                lab_err += "contain labels. Labels aren't supported yet."
-                                raise EasyBuildError(lab_err, name, filepath)
-                        else:
-                            versionsuffix = ''
-
-                        specs = {
-                            'name': name,
-                            'toolchain_name': toolchain_name,
-                            'toolchain_version': toolchain_version,
-                            'version': version,
-                            'versionsuffix': versionsuffix,
-                        }
-                        sw = SoftwareSpecs(**specs)
-
-                        # append newly created class instance to the list in instance of EasyStack class
-                        easystack.software_list.append(sw)
-                    continue
-
-                elif isinstance(versions, (list, tuple)):
-                    pass
-
-                # multiple lines without ':' is read as a single string; example:
-                # versions:
-                #   '2.24'
-                #   '2.51'
-                elif isinstance(versions, string_type):
-                    versions = versions.split()
-
-                # single values like '2.24' should be wrapped in a list
-                else:
-                    versions = [versions]
-
-                # if version is not a dictionary, versionsuffix is not specified
-                versionsuffix = ''
-
-                for version in versions:
-                    check_value(version, "%s (with %s toolchain)" % (name, toolchain_name))
-                    sw = SoftwareSpecs(
-                        name=name, version=version, versionsuffix=versionsuffix,
-                        toolchain_name=toolchain_name, toolchain_version=toolchain_version)
-                    # append newly created class instance to the list in instance of EasyStack class
-                    easystack.software_list.append(sw)
-
-            # assign general easystack attributes
-            easystack.easybuild_version = easybuild_version
-            easystack.robot = robot
-
-        return easystack
-
 
 @only_if_module_is_available('yaml', pkgname='PyYAML')
 def parse_easystack(filepath):
@@ -321,18 +195,17 @@ def parse_easystack(filepath):
     # class instance which contains all info about planned build
     easystack = EasyStackParser.parse(filepath)
 
-    easyconfig_names = easystack.compose_ec_filenames()
-
     # Disabled general options for now. We weren't using them, and first want support for EasyConfig-specific options.
     # Then, we need a method to resolve conflicts (specific options should win)
     # general_options = easystack.get_general_options()
 
-    _log.debug("EasyStack parsed. Proceeding to install these Easyconfigs: %s" % ', '.join(sorted(easyconfig_names)))
-    _log.debug("Using EasyConfig specific options based on the following dict:")
-    _log.debug(easystack.ec_opts)
+    _log.debug("Parsed easystack:\n%s" % easystack)
+
+#    _log.debug("Using EasyConfig specific options based on the following dict:")
+#    _log.debug(easystack.ec_opts)
     # if len(general_options) != 0:
     #    _log.debug("General options for installation are: \n%s" % str(general_options))
     # else:
     #    _log.debug("No general options were specified in easystack")
 
-    return easyconfig_names, easystack.ec_opts
+    return easystack