Merge pull request #1 from boegel/cfenoy_develop

sync with develop & resolve conflict

easybuild/framework/easyconfig/easyconfig.py

@@ -109,7 +109,8 @@ class EasyConfig(object):
     Class which handles loading, reading, validation of easyconfigs
     """
 
-    def __init__(self, path, extra_options=None, build_specs=None, validate=True, hidden=None, rawtxt=None):
+    def __init__(self, path, extra_options=None, build_specs=None, validate=True, hidden=None, rawtxt=None,
+                 auto_convert_value_types=True):
         """
         initialize an easyconfig.
         @param path: path to easyconfig file to be parsed (ignored if rawtxt is specified)
@@ -118,6 +119,8 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi
         @param validate: indicates whether validation should be performed (note: combined with 'validate' build option)
         @param hidden: indicate whether corresponding module file should be installed hidden ('.'-prefixed)
         @param rawtxt: raw contents of easyconfig file
+        @param auto_convert_value_types: indicates wether types of easyconfig values should be automatically converted
+                                         in case they are wrong
         """
         self.template_values = None
         self.enable_templating = True  # a boolean to control templating
@@ -184,7 +187,8 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi
 
         # parse easyconfig file
         self.build_specs = build_specs
-        self.parser = EasyConfigParser(filename=self.path, rawcontent=self.rawtxt)
+        self.parser = EasyConfigParser(filename=self.path, rawcontent=self.rawtxt,
+                                       auto_convert_value_types=auto_convert_value_types)
         self.parse()
 
         # handle allowed system dependencies
@@ -480,7 +484,21 @@ def toolchain(self):
         returns the Toolchain used
         """
         if self._toolchain is None:
-            self._toolchain = get_toolchain(self['toolchain'], self['toolchainopts'], mns=ActiveMNS())
+            # provide list of (direct) toolchain dependencies (name & version), if easyconfig can be found for toolchain
+            tcdeps = None
+            tcname, tcversion = self['toolchain']['name'], self['toolchain']['version']
+            if tcname != DUMMY_TOOLCHAIN_NAME:
+                tc_ecfile = robot_find_easyconfig(tcname, tcversion)
+                if tc_ecfile is None:
+                    self.log.debug("No easyconfig found for toolchain %s version %s, can't determine dependencies",
+                                   tcname, tcversion)
+                else:
+                    self.log.debug("Found easyconfig for toolchain %s version %s: %s", tcname, tcversion, tc_ecfile)
+                    tc_ec = process_easyconfig(tc_ecfile)[0]
+                    tcdeps = tc_ec['dependencies']
+                    self.log.debug("Toolchain dependencies based on easyconfig: %s", tcdeps)
+
+            self._toolchain = get_toolchain(self['toolchain'], self['toolchainopts'], mns=ActiveMNS(), tcdeps=tcdeps)
             tc_dict = self._toolchain.as_dict()
             self.log.debug("Initialized toolchain: %s (opts: %s)" % (tc_dict, self['toolchainopts']))
         return self._toolchain
@@ -1010,28 +1028,33 @@ def create_paths(path, name, version):
 
 def robot_find_easyconfig(name, version):
     """
-    Find an easyconfig for module in path
+    Find an easyconfig for module in path, returns (absolute) path to easyconfig file (or None, if none is found).
     """
     key = (name, version)
     if key in _easyconfig_files_cache:
         _log.debug("Obtained easyconfig path from cache for %s: %s" % (key, _easyconfig_files_cache[key]))
         return _easyconfig_files_cache[key]
+
     paths = build_option('robot_path')
-    if not paths:
-        raise EasyBuildError("No robot path specified, which is required when looking for easyconfigs (use --robot)")
-    if not isinstance(paths, (list, tuple)):
+    if paths is None:
+        paths = []
+    elif not isinstance(paths, (list, tuple)):
         paths = [paths]
-    # candidate easyconfig paths
+
+    res = None
     for path in paths:
         easyconfigs_paths = create_paths(path, name, version)
         for easyconfig_path in easyconfigs_paths:
             _log.debug("Checking easyconfig path %s" % easyconfig_path)
             if os.path.isfile(easyconfig_path):
                 _log.debug("Found easyconfig file for name %s, version %s at %s" % (name, version, easyconfig_path))
                 _easyconfig_files_cache[key] = os.path.abspath(easyconfig_path)
-                return _easyconfig_files_cache[key]
+                res = _easyconfig_files_cache[key]
+                break
+        if res:
+            break
 
-    return None
+    return res
 
 
 class ActiveMNS(object):

easybuild/framework/easyconfig/format/one.py

@@ -370,8 +370,8 @@ def retrieve_blocks_in_spec(spec, only_blocks, silent=False):
         ec_format_version = FORMAT_DEFAULT_VERSION
     _log.debug("retrieve_blocks_in_spec: derived easyconfig format version: %s" % ec_format_version)
 
-    # blocks in easyconfigs are only supported in format versions prior to 2.0
-    if pieces and ec_format_version < EasyVersion('2.0'):
+    # blocks in easyconfigs are only supported in easyconfig format 1.0
+    if pieces and ec_format_version == EasyVersion('1.0'):
         # make a map of blocks
         blocks = []
         while pieces:

easybuild/framework/easyconfig/format/yeb.py

@@ -0,0 +1,143 @@
+# #
+# Copyright 2013-2015 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en),
+# the Hercules foundation (http://www.herculesstichting.be/in_English)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# http://github.com/hpcugent/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+# #
+"""
+YAML easyconfig format (.yeb)
+Useful: http://www.yaml.org/spec/1.2/spec.html
+
+@author: Caroline De Brouwer (Ghent University)
+@author: Kenneth Hoste (Ghent University)
+"""
+import os
+from vsc.utils import fancylogger
+
+from easybuild.framework.easyconfig.format.format import INDENT_4SPACES, EasyConfigFormat
+from easybuild.framework.easyconfig.format.pyheaderconfigobj import build_easyconfig_constants_dict
+from easybuild.framework.easyconfig.format.pyheaderconfigobj import build_easyconfig_variables_dict
+from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.filetools import read_file
+from easybuild.tools.utilities import only_if_module_is_available, quote_str
+
+
+_log = fancylogger.getLogger('easyconfig.format.yeb', fname=False)
+
+
+YAML_DIR = r'%YAML'
+YAML_SEP = '---'
+YEB_FORMAT_EXTENSION = '.yeb'
+
+
+def yaml_join(loader, node):
+    """
+    defines custom YAML join function.
+    see http://stackoverflow.com/questions/5484016/how-can-i-do-string-concatenation-or-string-replacement-in-yaml/23212524#23212524
+    @param loader: the YAML Loader
+    @param node: the YAML (sequence) node
+    """
+    seq = loader.construct_sequence(node)
+    return ''.join([str(i) for i in seq])
+
+
+try:
+    import yaml
+    # register the tag handlers
+    yaml.add_constructor('!join', yaml_join)
+except ImportError:
+    pass
+
+
+class FormatYeb(EasyConfigFormat):
+    """Support for easyconfig YAML format"""
+    USABLE = True
+
+    def __init__(self):
+        """FormatYeb constructor"""
+        super(FormatYeb, self).__init__()
+        self.log.experimental("Parsing .yeb easyconfigs")
+
+    def validate(self):
+        """Format validation"""
+        _log.info(".yeb format validation isn't implemented (yet) - validation always passes")
+        return True
+
+    def get_config_dict(self):
+        """
+        Return parsed easyconfig as a dictionary, based on specified arguments.
+        """
+        return self.parsed_yeb
+
+    @only_if_module_is_available('yaml')
+    def parse(self, txt):
+        """
+        Process YAML file
+        """
+        txt = self._inject_constants_dict(txt)
+        self.parsed_yeb = yaml.load(txt)
+
+    def _inject_constants_dict(self, txt):
+        """Inject constants so they are resolved when actually parsing the YAML text."""
+        constants_dict = build_easyconfig_constants_dict()
+
+        lines = txt.splitlines()
+
+        # extract possible YAML header, for example
+        # %YAML 1.2
+        # ---
+        yaml_header = []
+        for i, line in enumerate(lines):
+            if line.startswith(YAML_DIR):
+                if lines[i+1].startswith(YAML_SEP):
+                    yaml_header.extend([lines.pop(i), lines.pop(i)])
+
+        injected_constants = ['__CONSTANTS__: ']
+        for key, value in constants_dict.items():
+            injected_constants.append('%s- &%s %s' % (INDENT_4SPACES, key, quote_str(value)))
+
+        full_txt = '\n'.join(yaml_header + injected_constants + lines)
+
+        return full_txt
+
+    def dump(self, ecfg, default_values, templ_const, templ_val):
+        """Dump parsed easyconfig in .yeb format"""
+        raise NotImplementedError("Dumping of .yeb easyconfigs not supported yet")
+
+    def extract_comments(self, txt):
+        """Extract comments from easyconfig file"""
+        self.log.debug("Not extracting comments from .yeb easyconfigs")
+
+
+def is_yeb_format(filename, rawcontent):
+    """
+    Determine whether easyconfig is in .yeb format.
+    If filename is None, rawcontent will be used to check the format.
+    """
+    isyeb = False
+    if filename:
+        isyeb = os.path.splitext(filename)[-1] == YEB_FORMAT_EXTENSION
+    else:
+        # if one line like 'name: ' is found, this must be YAML format
+        for line in rawcontent.splitlines():
+            if line.startswith('name: '):
+                isyeb = True
+    return isyeb

easybuild/framework/easyconfig/parser.py

@@ -35,6 +35,7 @@
 
 from easybuild.framework.easyconfig.format.format import FORMAT_DEFAULT_VERSION
 from easybuild.framework.easyconfig.format.format import get_format_version, get_format_version_classes
+from easybuild.framework.easyconfig.format.yeb import FormatYeb, is_yeb_format
 from easybuild.framework.easyconfig.types import TYPES, check_type_of_param_value
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.filetools import read_file, write_file
@@ -52,7 +53,6 @@
     'premakeopts': 'prebuildopts',
 }
 
-
 _log = fancylogger.getLogger('easyconfig.parser', fname=False)
 
 
@@ -64,7 +64,7 @@ def fetch_parameters_from_easyconfig(rawtxt, params):
     """
     param_values = []
     for param in params:
-        regex = re.compile(r"^\s*%s\s*=\s*(?P<param>\S.*?)\s*$" % param, re.M)
+        regex = re.compile(r"^\s*%s\s*(=|: )\s*(?P<param>\S.*?)\s*$" % param, re.M)
         res = regex.search(rawtxt)
         if res:
             param_values.append(res.group('param').strip("'\""))
@@ -79,21 +79,30 @@ class EasyConfigParser(object):
         Can contain references to multiple version and toolchain/toolchain versions
     """
 
-    def __init__(self, filename=None, format_version=None, rawcontent=None):
-        """Initialise the EasyConfigParser class"""
+    def __init__(self, filename=None, format_version=None, rawcontent=None,
+                 auto_convert_value_types=True):
+        """
+        Initialise the EasyConfigParser class
+        @param filename: path to easyconfig file to parse (superseded by rawcontent, if specified)
+        @param format_version: version of easyconfig file format, used to determine how to parse supplied easyconfig
+        @param rawcontent: raw content of easyconfig file to parse (preferred over easyconfig file supplied via filename)
+        @param auto_convert_value_types: indicates whether types of easyconfig values should be automatically converted
+                                         in case they are wrong
+        """
         self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
 
         self.rawcontent = None  # the actual unparsed content
 
+        self.auto_convert = auto_convert_value_types
+
         self.get_fn = None  # read method and args
         self.set_fn = None  # write method and args
 
         self.format_version = format_version
         self._formatter = None
-
         if rawcontent is not None:
             self.rawcontent = rawcontent
-            self._set_formatter()
+            self._set_formatter(filename)
         elif filename is not None:
             self._check_filename(filename)
             self.process()
@@ -105,7 +114,7 @@ def __init__(self, filename=None, format_version=None, rawcontent=None):
     def process(self, filename=None):
         """Create an instance"""
         self._read(filename=filename)
-        self._set_formatter()
+        self._set_formatter(filename)
 
     def check_values_types(self, cfg):
         """
@@ -115,9 +124,13 @@ def check_values_types(self, cfg):
         """
         wrong_type_msgs = []
         for key in cfg:
-            type_ok, _ = check_type_of_param_value(key, cfg[key])
+            type_ok, newval = check_type_of_param_value(key, cfg[key], self.auto_convert)
             if not type_ok:
                 wrong_type_msgs.append("value for '%s' should be of type '%s'" % (key, TYPES[key].__name__))
+            elif newval != cfg[key]:
+                self.log.warning("Value for '%s' easyconfig parameter was converted from %s (type: %s) to %s (type: %s)",
+                                 key, cfg[key], type(cfg[key]), newval, type(newval))
+                cfg[key] = newval
 
         if wrong_type_msgs:
             raise EasyBuildError("Type checking of easyconfig parameter values failed: %s", ', '.join(wrong_type_msgs))
@@ -172,11 +185,14 @@ def _get_format_version_class(self):
             raise EasyBuildError("More than one format class found matching version %s in %s",
                                  self.format_version, found_classes)
 
-    def _set_formatter(self):
+    def _set_formatter(self, filename):
         """Obtain instance of the formatter"""
         if self._formatter is None:
-            klass = self._get_format_version_class()
-            self._formatter = klass()
+            if is_yeb_format(filename, self.rawcontent):
+                self._formatter = FormatYeb()
+            else:
+                klass = self._get_format_version_class()
+                self._formatter = klass()
         self._formatter.parse(self.rawcontent)
 
     def set_format_text(self):

easybuild/framework/extension.py

@@ -37,7 +37,7 @@
 import os
 
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.config import build_path
+from easybuild.tools.config import build_option, build_path
 from easybuild.tools.run import run_cmd
 
 
@@ -62,7 +62,9 @@ def __init__(self, mself, ext):
         self.patches = self.ext.get('patches', [])
         self.options = copy.deepcopy(self.ext.get('options', {}))
 
-        self.toolchain.prepare(self.cfg['onlytcmod'])
+        # don't re-prepare the build environment when doing a dry run, since it'll be the same as for the parent
+        if not build_option('extended_dry_run'):
+            self.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True)
 
         self.sanity_check_fail_msgs = []
 

easybuild/main.py

@@ -129,7 +129,7 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
 
         # dump test report next to log file
         test_report_txt = create_test_report(test_msg, [(ec, ec_res)], init_session_state)
-        if 'log_file' in ec_res:
+        if 'log_file' in ec_res and ec_res['log_file']:
             test_report_fp = "%s_test_report.md" % '.'.join(ec_res['log_file'].split('.')[:-1])
             parent_dir = os.path.dirname(test_report_fp)
             # parent dir for test report may not be writable at this time, e.g. when --read-only-installdir is used
@@ -292,7 +292,7 @@ def main(args=None, logfile=None, do_build=None, testing=False):
         sys.exit(0)
 
     # skip modules that are already installed unless forced
-    if not options.force and not options.rebuild:
+    if not (options.force or options.rebuild or options.extended_dry_run):
         retained_ecs = skip_available(easyconfigs)
         if not testing:
             for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]:

easybuild/scripts/bootstrap_eb.py

@@ -331,7 +331,7 @@ def stage1(tmpdir, sourcepath):
     pattern = "This is EasyBuild (?P<version>%(v)s) \(framework: %(v)s, easyblocks: %(v)s\)" % {'v': '[0-9.]*[a-z0-9]*'}
     version_re = re.compile(pattern)
     version_out_file = os.path.join(tmpdir, 'eb_version.out')
-    eb_version_cmd = 'from easybuild.tools.version import this_is_easybuild; print this_is_easybuild()'
+    eb_version_cmd = 'from easybuild.tools.version import this_is_easybuild; print(this_is_easybuild())'
     cmd = "python -c '%s' > %s 2>&1" % (eb_version_cmd, version_out_file)
     debug("Determining EasyBuild version using command '%s'" % cmd)
     os.system(cmd)