Merge pull request #4 from deniskristak/feature/specsfile-devel

Feature/specsfile devel

easybuild/framework/easyconfig/easyconfig.py

@@ -1818,7 +1818,6 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error
             except ImportError as err:
                 # when an ImportError occurs, make sure that it's caused by not finding the easyblock module,
                 # and not because of a broken import statement in the easyblock module
-                print('modulepath' + modulepath)
                 modname = modulepath.replace('easybuild.easyblocks.', '')
                 error_re = re.compile(r"No module named '?.*/?%s'?" % modname)
                 _log.debug("error regexp for ImportError on '%s' easyblock: %s", modname, error_re.pattern)

easybuild/framework/easyconfig/tools.py

@@ -395,6 +395,7 @@ def parse_easyconfigs(paths, validate=True):
     """
     easyconfigs = []
     generated_ecs = False
+
     for (path, generated) in paths:
         path = os.path.abspath(path)
         # keep track of whether any files were generated

easybuild/main.py

@@ -70,7 +70,7 @@
 from easybuild.tools.parallelbuild import submit_jobs
 from easybuild.tools.repository.repository import init_repository
 from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state
-from easybuild.tools.build_from_specsfile import handle_specsfile
+from easybuild.tools.build_from_easystack import parse_easystack
 
 _log = None
 
@@ -224,6 +224,13 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
         last_log = find_last_log(logfile) or '(none)'
         print_msg(last_log, log=_log, prefix=False)
 
+    # if easystack is provided with the command, commands with arguments from it will be executed
+    if options.easystack:
+        # TODO add general_options (i.e. robot) to build options
+        orig_paths, general_options = parse_easystack(options.easystack)
+        if general_options:
+            raise EasyBuildError("Support for general options (flags) is not supported yet.")
+
     # check whether packaging is supported when it's being used
     if options.package:
         check_pkg_support()
@@ -322,12 +329,6 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
 
     # determine paths to easyconfigs
     determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs'])
-
-    # if specsfile is provided with the command, commands with arguments from it will be executed
-    if options.specsfile:
-        # TODO add general_options (i.e. robot) to build options
-        determined_paths, general_options = handle_specsfile(options.specsfile)
-
     if (options.copy_ec and not tweaked_ecs_paths) or options.fix_deprecated_easyconfigs or options.show_ec:
 
         if options.copy_ec:
@@ -422,7 +423,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
             ordered_ecs = resolve_dependencies(easyconfigs, modtool)
         else:
             ordered_ecs = easyconfigs
-    elif pr_options or options.specsfile:
+    elif pr_options:
         ordered_ecs = None
     else:
         print_msg("No easyconfigs left to be built.", log=_log, silent=testing)

easybuild/tools/build_from_specsfile.py → easybuild/tools/build_from_easystack.py

@@ -4,45 +4,34 @@
 from easybuild.base import fancylogger
 
 
-_log = fancylogger.getLogger('specsfile', fname=False)
+_log = fancylogger.getLogger('easystack', fname=False)
 
 
 # general specs applicable to all commands
-class Specsfile(object):
+class Easystack(object):
     def __init__(self):
         self.easybuild_version = None
         self.robot = False
         self.software_list = []
 
-    # returns list of all commands - finished
-    def compose_full_paths(self):
-        easyconfigs_full_paths = []
+    # returns list of all easyconfig names - finished
+    def compose_ec_names(self):
+        ec_names = []
         for sw in self.software_list:
-            path_to_append = self.get_ec_path(sw)
-
-            if path_to_append is None:
+            ec_to_append = '%s-%s-%s-%s.eb' % (str(sw.software), str(sw.version),
+                                            str(sw.toolchain_name), str(sw.toolchain_version))
+            if ec_to_append is None:
                 continue
             else:
-                easyconfigs_full_paths.append(path_to_append)
-        return easyconfigs_full_paths
-
-    # single command
-    def get_ec_path(self, sw):
-        full_path = search_easyconfigs(query='%s-%s-%s-%s' % (
-            str(sw.software), str(sw.version), str(sw.toolchain_name), str(sw.toolchain_version)),
-            short=False, filename_only=False,
-            terse=False, consider_extra_paths=True, print_result=False, case_sensitive=False)
-        if len(full_path) == 1:
-            return full_path[0]
-        else:
-            print('%s does not have clearly specified parameters - %s matches found. Skipping. \n' %
-                  (sw.software, str(len(full_path))))
-
-    # todo: flags applicable to all sw (i.e. robot)
+                ec_names.append(ec_to_append)
+        return ec_names
+
+    # flags applicable to all sw (i.e. robot)
     def get_general_options(self):
         general_options = {}
-        general_options['robot'] = self.robot
-        general_options['easybuild_version'] = self.easybuild_version
+        # TODO add support for general_options
+        # general_options['robot'] = self.robot
+        # general_options['easybuild_version'] = self.easybuild_version
         return general_options
 
 
@@ -56,30 +45,30 @@ def __init__(self, software, version, toolchain, toolchain_version, toolchain_na
         self.toolchain_name = toolchain_name
         self.toolchain = toolchain
 
-        self.version_suffix = None
+        self.versionsuffix = None
 
-    def get_version_suffix(self):
-        return self.version_suffix or ''
+    def get_versionsuffix(self):
+        return self.versionsuffix or ''
 
 
 # implement this to your own needs - to create custom yaml/json/xml parser
 class GenericSpecsParser(object):
-    @staticmethod
+    @ staticmethod
     def parse(filename):
         raise NotImplementedError
 
 
 class YamlSpecParser(GenericSpecsParser):
-    @staticmethod
+    @ staticmethod
     def parse(filename):
 
         try:
             with open(filename, 'r') as f:
                 spec_dict = yaml.safe_load(f)
 
-            eb = Specsfile()
+            eb = Easystack()
         except FileNotFoundError:
-            raise EasyBuildError("Could not read provided specsfile.")
+            raise EasyBuildError("Could not read provided easystack.")
 
         sw_dict = spec_dict["software"]
 
@@ -120,31 +109,28 @@ def parse(filename):
                         elif str(yaml_version) == 'exclude-labels' \
                                 or str(yaml_version) == 'include-labels':
                             continue
-
                         else:
-                            print('Software % s has wrong yaml structure!' % (str(software)))
-                            raise EasyBuildError('Wrong yaml structure')
+                            raise EasyBuildError('Software % s has wrong yaml structure!' % (str(software)))
 
             except (KeyError, TypeError, IndexError):
-                print('Software % s has wrong yaml structure!' % (str(software)))
-                raise EasyBuildError('Wrong yaml structure')
+                raise EasyBuildError('Software % s has wrong yaml structure!' % (str(software)))
 
         # assign general EB attributes
         eb.easybuild_version = spec_dict.get('easybuild_version', None)
         eb.robot = spec_dict.get('robot', False)
         return eb
 
 
-def handle_specsfile(filename):
-    _log.info("Building from specsfile: '%s'" % filename)
+def parse_easystack(filename):
+    _log.info("Building from easystack: '%s'" % filename)
 
     # class instance which contains all info about planned build
     eb = YamlSpecParser.parse(filename)
 
-    easyconfigs_full_paths = eb.compose_full_paths()
+    easyconfigs_full_paths = eb.compose_ec_names()
 
     general_options = eb.get_general_options()
 
-    _log.debug("Specsfile parsed. Proceeding to install these Easyconfigs: \n'%s'" % ',\n'.join(easyconfigs_full_paths))
+    _log.debug("Easystack parsed. Proceeding to install these Easyconfigs: \n'%s'" % ',\n'.join(easyconfigs_full_paths))
 
     return easyconfigs_full_paths, general_options

easybuild/tools/options.py

@@ -604,8 +604,8 @@ def informative_options(self):
             'show-full-config': ("Show current EasyBuild configuration (all settings)", None, 'store_true', False),
             'show-system-info': ("Show system information relevant to EasyBuild", None, 'store_true', False),
             'terse': ("Terse output (machine-readable)", None, 'store_true', False),
-            'specsfile': (
-                "Accepts file containing build specifications (i.e. yaml), parses it and prints all eb commands to run",
+            'easystack': (
+                "Parses file containing build specifications (i.e. yaml) and initiates build accordingly",
                 None, 'store', None),
         })
 

test/framework/options.py

@@ -5453,24 +5453,24 @@ def test_sysroot(self):
         os.environ['EASYBUILD_SYSROOT'] = doesnotexist
         self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, ['--show-config'], raise_error=True)
 
-    def test_wrong_read_specsfile(self):
-        """Test for --specsfile <specsfile.yaml> when wrong name is provided"""
+    def test_wrong_read_easystack(self):
+        """Test for --easystack <easystack.yaml> when wrong name is provided"""
         topdir = os.path.dirname(os.path.abspath(__file__))
-        toy_specsfile = os.path.join(topdir, 'specsfiles', 'test_specsfile_nonexistent.yaml')
+        toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_nonexistent.yaml')
 
-        args = ['--specsfile', toy_specsfile]
+        args = ['--easystack', toy_easystack]
         self.mock_stdout(True)
         self.mock_stderr(True)
-        self.assertErrorRegex(EasyBuildError, "Could not read provided specsfile", self.eb_main, args, raise_error=True)
+        self.assertErrorRegex(EasyBuildError, "Could not read provided easystack", self.eb_main, args, raise_error=True)
         self.mock_stdout(False)
         self.mock_stderr(False)
 
-    def test_wrong_specsfile_structure(self):
-        """Test for --specsfile <specsfile.yaml> when yaml specsfile has wrong structure"""
+    def test_wrong_easystack_structure(self):
+        """Test for --easystack <easystack.yaml> when yaml easystack has wrong structure"""
         topdir = os.path.dirname(os.path.abspath(__file__))
-        toy_specsfile = os.path.join(topdir, 'specsfiles', 'test_specsfile_wrong_structure.yaml')
+        toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_wrong_structure.yaml')
 
-        args = ['--specsfile', toy_specsfile]
+        args = ['--easystack', toy_easystack]
         self.mock_stdout(True)
         self.mock_stderr(True)
         self.assertErrorRegex(EasyBuildError, "Wrong yaml structure", self.eb_main, args, raise_error=True)
@@ -5480,12 +5480,12 @@ def test_wrong_specsfile_structure(self):
     # basic stuff in yaml - just to test the basics
     # no implicit dependencies - all listed in yaml (thus no need for --robot
     # expecting successful build
-    def test_specsfile_basic(self):
-        """Test for --specsfile <specsfile.yaml> -> success case"""
+    def test_basic_easystack(self):
+        """Test for --easystack <easystack.yaml> -> success case"""
         topdir = os.path.dirname(os.path.abspath(__file__))
-        toy_specsfile = os.path.join(topdir, 'specsfiles', 'test_specsfile_basic.yaml')
+        toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_basic.yaml')
 
-        args = ['--specsfile', toy_specsfile, '--stop', '--debug']
+        args = ['--easystack', toy_easystack, '--stop', '--debug']
         stdout, err = self.eb_main(args, do_build=True, return_error=True, testing=True)
         print(stdout)
 
@@ -5496,12 +5496,12 @@ def test_specsfile_basic(self):
     # basic stuff in yaml
     # `--robot` is added and dependencies are not listed
     # expecting successful build
-    def test_specsfile_robot(self):
-        """Test for --specsfile <specsfile.yaml> -> success case"""
+    def test_easystack_robot(self):
+        """Test for --easystack <easystack.yaml> -> success case"""
         topdir = os.path.dirname(os.path.abspath(__file__))
-        toy_specsfile = os.path.join(topdir, 'specsfiles', 'test_specsfile_robot.yaml')
+        toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_robot.yaml')
 
-        args = ['--specsfile', toy_specsfile, '--stop', '--robot', '--debug']
+        args = ['--easystack', toy_easystack, '--stop', '--debug']
         stdout, err = self.eb_main(args, do_build=True, return_error=True)
         print(stdout)