From 36117c5fc6eafa41b7b16c5d8d0d3170a0398f8a Mon Sep 17 00:00:00 2001 From: Denis Kristak Date: Thu, 5 Nov 2020 12:57:43 +0100 Subject: [PATCH 1/3] rename to easystack --- easybuild/main.py | 10 +++--- ...m_specsfile.py => build_from_easystack.py} | 12 +++---- easybuild/tools/options.py | 2 +- test/framework/options.py | 34 +++++++++---------- ...specsfile_1.yaml => test_easystack_1.yaml} | 0 ...e_basic.yaml => test_easystack_basic.yaml} | 0 ...e_robot.yaml => test_easystack_robot.yaml} | 0 ...ml => test_easystack_wrong_structure.yaml} | 0 8 files changed, 29 insertions(+), 29 deletions(-) rename easybuild/tools/{build_from_specsfile.py => build_from_easystack.py} (95%) rename test/framework/specsfiles/{test_specsfile_1.yaml => test_easystack_1.yaml} (100%) rename test/framework/specsfiles/{test_specsfile_basic.yaml => test_easystack_basic.yaml} (100%) rename test/framework/specsfiles/{test_specsfile_robot.yaml => test_easystack_robot.yaml} (100%) rename test/framework/specsfiles/{test_specsfile_wrong_structure.yaml => test_easystack_wrong_structure.yaml} (100%) diff --git a/easybuild/main.py b/easybuild/main.py index ff4d74d584..c6c50db130 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -70,7 +70,7 @@ from easybuild.tools.parallelbuild import submit_jobs from easybuild.tools.repository.repository import init_repository from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state -from easybuild.tools.build_from_specsfile import handle_specsfile +from easybuild.tools.build_from_easystack import handle_easystack _log = None @@ -323,10 +323,10 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # determine paths to easyconfigs determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) - # if specsfile is provided with the command, commands with arguments from it will be executed - if options.specsfile: + # if easystack is provided with the command, commands with arguments from it will be executed + if options.easystack: # TODO add general_options (i.e. robot) to build options - determined_paths, general_options = handle_specsfile(options.specsfile) + determined_paths, general_options = handle_easystack(options.easystack) if (options.copy_ec and not tweaked_ecs_paths) or options.fix_deprecated_easyconfigs or options.show_ec: @@ -422,7 +422,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): ordered_ecs = resolve_dependencies(easyconfigs, modtool) else: ordered_ecs = easyconfigs - elif pr_options or options.specsfile: + elif pr_options or options.easystack: ordered_ecs = None else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) diff --git a/easybuild/tools/build_from_specsfile.py b/easybuild/tools/build_from_easystack.py similarity index 95% rename from easybuild/tools/build_from_specsfile.py rename to easybuild/tools/build_from_easystack.py index 6f54e45903..613db4414b 100644 --- a/easybuild/tools/build_from_specsfile.py +++ b/easybuild/tools/build_from_easystack.py @@ -4,11 +4,11 @@ from easybuild.base import fancylogger -_log = fancylogger.getLogger('specsfile', fname=False) +_log = fancylogger.getLogger('easystack', fname=False) # general specs applicable to all commands -class Specsfile(object): +class Easystack(object): def __init__(self): self.easybuild_version = None self.robot = False @@ -77,9 +77,9 @@ def parse(filename): with open(filename, 'r') as f: spec_dict = yaml.safe_load(f) - eb = Specsfile() + eb = Easystack() except FileNotFoundError: - raise EasyBuildError("Could not read provided specsfile.") + raise EasyBuildError("Could not read provided easystack.") sw_dict = spec_dict["software"] @@ -135,8 +135,8 @@ def parse(filename): return eb -def handle_specsfile(filename): - _log.info("Building from specsfile: '%s'" % filename) +def handle_easystack(filename): + _log.info("Building from easystack: '%s'" % filename) # class instance which contains all info about planned build eb = YamlSpecParser.parse(filename) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 80c3255eb3..e2b1528657 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -604,7 +604,7 @@ def informative_options(self): 'show-full-config': ("Show current EasyBuild configuration (all settings)", None, 'store_true', False), 'show-system-info': ("Show system information relevant to EasyBuild", None, 'store_true', False), 'terse': ("Terse output (machine-readable)", None, 'store_true', False), - 'specsfile': ( + 'easystack': ( "Accepts file containing build specifications (i.e. yaml), parses it and prints all eb commands to run", None, 'store', None), }) diff --git a/test/framework/options.py b/test/framework/options.py index 4dacde2051..40d198c78c 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5453,24 +5453,24 @@ def test_sysroot(self): os.environ['EASYBUILD_SYSROOT'] = doesnotexist self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, ['--show-config'], raise_error=True) - def test_wrong_read_specsfile(self): - """Test for --specsfile when wrong name is provided""" + def test_wrong_read_easystack(self): + """Test for --easystack when wrong name is provided""" topdir = os.path.dirname(os.path.abspath(__file__)) - toy_specsfile = os.path.join(topdir, 'specsfiles', 'test_specsfile_nonexistent.yaml') + toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_nonexistent.yaml') - args = ['--specsfile', toy_specsfile] + args = ['--easystack', toy_easystack] self.mock_stdout(True) self.mock_stderr(True) - self.assertErrorRegex(EasyBuildError, "Could not read provided specsfile", self.eb_main, args, raise_error=True) + self.assertErrorRegex(EasyBuildError, "Could not read provided easystack", self.eb_main, args, raise_error=True) self.mock_stdout(False) self.mock_stderr(False) - def test_wrong_specsfile_structure(self): - """Test for --specsfile when yaml specsfile has wrong structure""" + def test_wrong_easystack_structure(self): + """Test for --easystack when yaml easystack has wrong structure""" topdir = os.path.dirname(os.path.abspath(__file__)) - toy_specsfile = os.path.join(topdir, 'specsfiles', 'test_specsfile_wrong_structure.yaml') + toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_wrong_structure.yaml') - args = ['--specsfile', toy_specsfile] + args = ['--easystack', toy_easystack] self.mock_stdout(True) self.mock_stderr(True) self.assertErrorRegex(EasyBuildError, "Wrong yaml structure", self.eb_main, args, raise_error=True) @@ -5480,12 +5480,12 @@ def test_wrong_specsfile_structure(self): # basic stuff in yaml - just to test the basics # no implicit dependencies - all listed in yaml (thus no need for --robot # expecting successful build - def test_basic_specsfile(self): - """Test for --specsfile -> success case""" + def test_basic_easystack(self): + """Test for --easystack -> success case""" topdir = os.path.dirname(os.path.abspath(__file__)) - toy_specsfile = os.path.join(topdir, 'specsfiles', 'test_specsfile_basic.yaml') + toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_basic.yaml') - args = ['--specsfile', toy_specsfile, '--stop', '--debug'] + args = ['--easystack', toy_easystack, '--stop', '--debug'] stdout, err = self.eb_main(args, do_build=True, return_error=True, testing=True) print(stdout) @@ -5496,12 +5496,12 @@ def test_basic_specsfile(self): # basic stuff in yaml # `--robot` is added and dependencies are not listed # expecting successful build - def test_specsfile_robot(self): - """Test for --specsfile -> success case""" + def test_easystack_robot(self): + """Test for --easystack -> success case""" topdir = os.path.dirname(os.path.abspath(__file__)) - toy_specsfile = os.path.join(topdir, 'specsfiles', 'test_specsfile_robot.yaml') + toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_robot.yaml') - args = ['--specsfile', toy_specsfile, '--stop', '--debug'] + args = ['--easystack', toy_easystack, '--stop', '--debug'] stdout, err = self.eb_main(args, do_build=True, return_error=True) print(stdout) diff --git a/test/framework/specsfiles/test_specsfile_1.yaml b/test/framework/specsfiles/test_easystack_1.yaml similarity index 100% rename from test/framework/specsfiles/test_specsfile_1.yaml rename to test/framework/specsfiles/test_easystack_1.yaml diff --git a/test/framework/specsfiles/test_specsfile_basic.yaml b/test/framework/specsfiles/test_easystack_basic.yaml similarity index 100% rename from test/framework/specsfiles/test_specsfile_basic.yaml rename to test/framework/specsfiles/test_easystack_basic.yaml diff --git a/test/framework/specsfiles/test_specsfile_robot.yaml b/test/framework/specsfiles/test_easystack_robot.yaml similarity index 100% rename from test/framework/specsfiles/test_specsfile_robot.yaml rename to test/framework/specsfiles/test_easystack_robot.yaml diff --git a/test/framework/specsfiles/test_specsfile_wrong_structure.yaml b/test/framework/specsfiles/test_easystack_wrong_structure.yaml similarity index 100% rename from test/framework/specsfiles/test_specsfile_wrong_structure.yaml rename to test/framework/specsfiles/test_easystack_wrong_structure.yaml From 64879f5f38a17e9d485861762ad4e95f61cfd05b Mon Sep 17 00:00:00 2001 From: Denis Kristak Date: Thu, 5 Nov 2020 13:41:34 +0100 Subject: [PATCH 2/3] test --- easybuild/main.py | 13 +++++--- easybuild/tools/build_from_easystack.py | 42 +++++++++---------------- 2 files changed, 23 insertions(+), 32 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index c6c50db130..97a2af9dc9 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -224,6 +224,14 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): last_log = find_last_log(logfile) or '(none)' print_msg(last_log, log=_log, prefix=False) + # if easystack is provided with the command, commands with arguments from it will be executed + if options.easystack: + # TODO add general_options (i.e. robot) to build options + orig_paths, general_options = handle_easystack(options.easystack) + print(orig_paths) + if general_options is not None: + raise EasyBuildError("Support for general options (flags) is not supported yet.") + # check whether packaging is supported when it's being used if options.package: check_pkg_support() @@ -323,11 +331,6 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # determine paths to easyconfigs determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) - # if easystack is provided with the command, commands with arguments from it will be executed - if options.easystack: - # TODO add general_options (i.e. robot) to build options - determined_paths, general_options = handle_easystack(options.easystack) - if (options.copy_ec and not tweaked_ecs_paths) or options.fix_deprecated_easyconfigs or options.show_ec: if options.copy_ec: diff --git a/easybuild/tools/build_from_easystack.py b/easybuild/tools/build_from_easystack.py index 91541bb668..6605ba6631 100644 --- a/easybuild/tools/build_from_easystack.py +++ b/easybuild/tools/build_from_easystack.py @@ -14,31 +14,19 @@ def __init__(self): self.robot = False self.software_list = [] - # returns list of all commands - finished - def compose_full_paths(self): - easyconfigs_full_paths = [] + # returns list of all easyconfig names - finished + def compose_ec_names(self): + ec_names = [] for sw in self.software_list: - path_to_append = self.get_ec_path(sw) - - if path_to_append is None: + ec_to_append = '%s-%s-%s-%s' % (str(sw.software), str(sw.version), + str(sw.toolchain_name), str(sw.toolchain_version)) + if ec_to_append is None: continue else: - easyconfigs_full_paths.append(path_to_append) - return easyconfigs_full_paths - - # single command - def get_ec_path(self, sw): - full_path = search_easyconfigs(query='%s-%s-%s-%s' % ( - str(sw.software), str(sw.version), str(sw.toolchain_name), str(sw.toolchain_version)), - short=False, filename_only=False, - terse=False, consider_extra_paths=True, print_result=False, case_sensitive=False) - if len(full_path) == 1: - return full_path[0] - else: - print('%s does not have clearly specified parameters - %s matches found. Skipping. \n' % - (sw.software, str(len(full_path)))) - - # todo: flags applicable to all sw (i.e. robot) + ec_names.append(ec_to_append) + return ec_names + + # flags applicable to all sw (i.e. robot) def get_general_options(self): general_options = {} general_options['robot'] = self.robot @@ -56,21 +44,21 @@ def __init__(self, software, version, toolchain, toolchain_version, toolchain_na self.toolchain_name = toolchain_name self.toolchain = toolchain - self.version_suffix = None + self.versionsuffix = None - def get_version_suffix(self): - return self.version_suffix or '' + def get_versionsuffix(self): + return self.versionsuffix or '' # implement this to your own needs - to create custom yaml/json/xml parser class GenericSpecsParser(object): - @staticmethod + @ staticmethod def parse(filename): raise NotImplementedError class YamlSpecParser(GenericSpecsParser): - @staticmethod + @ staticmethod def parse(filename): try: From 054327570910872893868d5047ca2a5f52105f2f Mon Sep 17 00:00:00 2001 From: deniskristak Date: Thu, 5 Nov 2020 14:27:59 +0100 Subject: [PATCH 3/3] test --- easybuild/framework/easyconfig/easyconfig.py | 1 - easybuild/framework/easyconfig/tools.py | 1 + easybuild/main.py | 10 ++++------ easybuild/tools/build_from_easystack.py | 18 ++++++++---------- easybuild/tools/options.py | 2 +- 5 files changed, 14 insertions(+), 18 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index c3ffc7a380..a0d47dd086 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1818,7 +1818,6 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error except ImportError as err: # when an ImportError occurs, make sure that it's caused by not finding the easyblock module, # and not because of a broken import statement in the easyblock module - print('modulepath' + modulepath) modname = modulepath.replace('easybuild.easyblocks.', '') error_re = re.compile(r"No module named '?.*/?%s'?" % modname) _log.debug("error regexp for ImportError on '%s' easyblock: %s", modname, error_re.pattern) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index ade0954a40..2a3260ae80 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -395,6 +395,7 @@ def parse_easyconfigs(paths, validate=True): """ easyconfigs = [] generated_ecs = False + for (path, generated) in paths: path = os.path.abspath(path) # keep track of whether any files were generated diff --git a/easybuild/main.py b/easybuild/main.py index 97a2af9dc9..6f09c9f938 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -70,7 +70,7 @@ from easybuild.tools.parallelbuild import submit_jobs from easybuild.tools.repository.repository import init_repository from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state -from easybuild.tools.build_from_easystack import handle_easystack +from easybuild.tools.build_from_easystack import parse_easystack _log = None @@ -227,9 +227,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # if easystack is provided with the command, commands with arguments from it will be executed if options.easystack: # TODO add general_options (i.e. robot) to build options - orig_paths, general_options = handle_easystack(options.easystack) - print(orig_paths) - if general_options is not None: + orig_paths, general_options = parse_easystack(options.easystack) + if general_options: raise EasyBuildError("Support for general options (flags) is not supported yet.") # check whether packaging is supported when it's being used @@ -330,7 +329,6 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # determine paths to easyconfigs determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) - if (options.copy_ec and not tweaked_ecs_paths) or options.fix_deprecated_easyconfigs or options.show_ec: if options.copy_ec: @@ -425,7 +423,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): ordered_ecs = resolve_dependencies(easyconfigs, modtool) else: ordered_ecs = easyconfigs - elif pr_options or options.easystack: + elif pr_options: ordered_ecs = None else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) diff --git a/easybuild/tools/build_from_easystack.py b/easybuild/tools/build_from_easystack.py index 6605ba6631..ab5fd27aa9 100644 --- a/easybuild/tools/build_from_easystack.py +++ b/easybuild/tools/build_from_easystack.py @@ -18,7 +18,7 @@ def __init__(self): def compose_ec_names(self): ec_names = [] for sw in self.software_list: - ec_to_append = '%s-%s-%s-%s' % (str(sw.software), str(sw.version), + ec_to_append = '%s-%s-%s-%s.eb' % (str(sw.software), str(sw.version), str(sw.toolchain_name), str(sw.toolchain_version)) if ec_to_append is None: continue @@ -29,8 +29,9 @@ def compose_ec_names(self): # flags applicable to all sw (i.e. robot) def get_general_options(self): general_options = {} - general_options['robot'] = self.robot - general_options['easybuild_version'] = self.easybuild_version + # TODO add support for general_options + # general_options['robot'] = self.robot + # general_options['easybuild_version'] = self.easybuild_version return general_options @@ -108,14 +109,11 @@ def parse(filename): elif str(yaml_version) == 'exclude-labels' \ or str(yaml_version) == 'include-labels': continue - else: - print('Software % s has wrong yaml structure!' % (str(software))) - raise EasyBuildError('Wrong yaml structure') + raise EasyBuildError('Software % s has wrong yaml structure!' % (str(software))) except (KeyError, TypeError, IndexError): - print('Software % s has wrong yaml structure!' % (str(software))) - raise EasyBuildError('Wrong yaml structure') + raise EasyBuildError('Software % s has wrong yaml structure!' % (str(software))) # assign general EB attributes eb.easybuild_version = spec_dict.get('easybuild_version', None) @@ -123,13 +121,13 @@ def parse(filename): return eb -def handle_easystack(filename): +def parse_easystack(filename): _log.info("Building from easystack: '%s'" % filename) # class instance which contains all info about planned build eb = YamlSpecParser.parse(filename) - easyconfigs_full_paths = eb.compose_full_paths() + easyconfigs_full_paths = eb.compose_ec_names() general_options = eb.get_general_options() diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index e2b1528657..56106995c7 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -605,7 +605,7 @@ def informative_options(self): 'show-system-info': ("Show system information relevant to EasyBuild", None, 'store_true', False), 'terse': ("Terse output (machine-readable)", None, 'store_true', False), 'easystack': ( - "Accepts file containing build specifications (i.e. yaml), parses it and prints all eb commands to run", + "Parses file containing build specifications (i.e. yaml) and initiates build accordingly", None, 'store', None), })