From 84f125aed13fa5f1db5eddf8dfaab5ffa15f0297 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 8 Jun 2015 16:01:57 +0200 Subject: [PATCH] make unit test suite pass, rename JobServer to JobBackend, fix default for --job-bakcend --- easybuild/tools/config.py | 3 +-- easybuild/tools/job/__init__.py | 20 ++-------------- easybuild/tools/job/gc3pie.py | 6 ++--- easybuild/tools/job/pbs_python.py | 13 ++++++---- easybuild/tools/options.py | 13 +++++----- easybuild/tools/parallelbuild.py | 8 +++---- test/framework/parallelbuild.py | 40 +++++++++++++++++++++---------- 7 files changed, 50 insertions(+), 53 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index e0352afd5f..a3199dadd8 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -52,6 +52,7 @@ _log = fancylogger.getLogger('config', fname=False) +DEFAULT_JOB_BACKEND = 'PbsPython' DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log") DEFAULT_MNS = 'EasyBuildMNS' DEFAULT_MODULE_SYNTAX = 'Tcl' @@ -68,8 +69,6 @@ DEFAULT_REPOSITORY = 'FileRepository' DEFAULT_STRICT = run.WARN -PREFERRED_JOB_BACKENDS = ('Pbs', 'GC3Pie') - # utility function for obtaining default paths def mk_full_default_path(name, prefix=DEFAULT_PREFIX): diff --git a/easybuild/tools/job/__init__.py b/easybuild/tools/job/__init__.py index 5d11408053..194e469358 100644 --- a/easybuild/tools/job/__init__.py +++ b/easybuild/tools/job/__init__.py @@ -30,11 +30,10 @@ from vsc.utils.missing import get_subclasses from easybuild.tools.config import get_job_backend -from easybuild.tools.config import PREFERRED_JOB_BACKENDS from easybuild.tools.utilities import import_available_modules -class JobServer(object): +class JobBackend(object): __metaclass__ = ABCMeta USABLE = False @@ -92,9 +91,7 @@ def avail_job_backends(check_usable=True): Return all known job execution backends. """ import_available_modules('easybuild.tools.job') - class_dict = dict([(x.__name__, x) - for x in get_subclasses(JobServer) - if (x.USABLE or not check_usable)]) + class_dict = dict([(x.__name__, x) for x in get_subclasses(JobBackend)]) return class_dict @@ -107,16 +104,3 @@ def job_backend(): return None job_backend_class = avail_job_backends().get(job_backend) return job_backend_class() - - -def preferred_job_backend(order=PREFERRED_JOB_BACKENDS): - """ - Return name of preferred concrete `JobServer` instance, or `None` - if none is available. - """ - available_backends = avail_job_backends() - for backend in order: - if backend in available_backends: - return backend - break - return None diff --git a/easybuild/tools/job/gc3pie.py b/easybuild/tools/job/gc3pie.py index 62ebfb3523..ecf375dda3 100644 --- a/easybuild/tools/job/gc3pie.py +++ b/easybuild/tools/job/gc3pie.py @@ -44,7 +44,7 @@ HAVE_GC3PIE = False from easybuild.tools.build_log import print_msg -from easybuild.tools.job import JobServer +from easybuild.tools.job import JobBackend from vsc.utils import fancylogger @@ -56,7 +56,7 @@ # eb --job --job-backend=GC3Pie -class GC3Pie(JobServer): +class GC3Pie(JobBackend): """ Use the GC3Pie__ framework to submit and monitor compilation jobs. @@ -83,7 +83,7 @@ def make_job(self, script, name, env_vars=None, hours=None, cores=None): Create and return a job object with the given parameters. First argument `server` is an instance of the corresponding - `JobServer` class, i.e., a `GC3Pie`:class: instance in this case. + `JobBackend` class, i.e., a `GC3Pie`:class: instance in this case. Second argument `script` is the content of the job script itself, i.e., the sequence of shell commands that will be diff --git a/easybuild/tools/job/pbs_python.py b/easybuild/tools/job/pbs_python.py index 15323d4f5b..e334157199 100644 --- a/easybuild/tools/job/pbs_python.py +++ b/easybuild/tools/job/pbs_python.py @@ -36,7 +36,7 @@ from vsc.utils import fancylogger from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.job import JobServer +from easybuild.tools.job import JobBackend _log = fancylogger.getLogger('pbs_python', fname=False) @@ -58,27 +58,30 @@ def only_if_pbs_import_successful(fn): HAVE_PBS_PYTHON = True except ImportError: _log.debug("Failed to import pbs from pbs_python." - " Silently ignoring, this is a real issue only with --job=pbs") + " Silently ignoring, this is a real issue only when pbs_python is used as backend for --job") # no `pbs_python` available, turn function into a no-op def only_if_pbs_import_successful(fn): def instead(*args, **kwargs): """This is a no-op since `pbs_python` is not available.""" errmsg = ("PBSQuery or pbs modules not available." " Please make sure `pbs_python` is installed and usable.") - _log.error(errmsg) - raise RuntimeError(errmsg) + raise EasyBuildError(errmsg) return instead HAVE_PBS_PYTHON = False -class Pbs(JobServer): +class PbsPython(JobBackend): """ Manage PBS server communication and create `PbsJob` objects. """ USABLE = HAVE_PBS_PYTHON + @only_if_pbs_import_successful def __init__(self, pbs_server=None): + _init() + + def _init(self, pbs_server=None): self.pbs_server = pbs_server or pbs.pbs_default() self.conn = None self._ppn = None diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 5aa5e2cbf8..8780b2c12e 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -48,16 +48,15 @@ from easybuild.framework.easyconfig.templates import template_documentation from easybuild.framework.easyconfig.tools import get_paths_for from easybuild.framework.extension import Extension -from easybuild.tools import build_log, config, run # @UnusedImport make sure config is always initialized! +from easybuild.tools import build_log, config, run # build_log should always stay there, to ensure EasyBuildLog from easybuild.tools.build_log import EasyBuildError, raise_easybuilderror -from easybuild.tools.config import DEFAULT_LOGFILE_FORMAT, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL -from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PREFIX, DEFAULT_REPOSITORY -from easybuild.tools.config import DEFAULT_STRICT, get_pretend_installpath, mk_full_default_path -from easybuild.tools.config import PREFERRED_JOB_BACKENDS +from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX +from easybuild.tools.config import DEFAULT_MODULES_TOOL, DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PREFIX +from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_STRICT, get_pretend_installpath, mk_full_default_path from easybuild.tools.configobj import ConfigObj, ConfigObjError from easybuild.tools.docs import FORMAT_RST, FORMAT_TXT, avail_easyconfig_params from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, fetch_github_token -from easybuild.tools.job import avail_job_backends, preferred_job_backend +from easybuild.tools.job import avail_job_backends from easybuild.tools.modules import avail_modules_tools from easybuild.tools.module_generator import ModuleGeneratorLua, avail_module_generators from easybuild.tools.module_naming_scheme import GENERAL_CLASS @@ -249,7 +248,7 @@ def config_options(self): 'installpath-software': ("Install path for software (if None, combine --installpath and --subdir-software)", None, 'store', None), 'job-backend': ("What job runner to use", 'choice', 'store', - preferred_job_backend(), (avail_job_backends().keys())), + DEFAULT_JOB_BACKEND, avail_job_backends().keys()), # purposely take a copy for the default logfile format 'logfile-format': ("Directory name and format of the log file", 'strtuple', 'store', DEFAULT_LOGFILE_FORMAT[:], {'metavar': 'DIR,FORMAT'}), diff --git a/easybuild/tools/parallelbuild.py b/easybuild/tools/parallelbuild.py index 80f952705d..bc42d775a4 100644 --- a/easybuild/tools/parallelbuild.py +++ b/easybuild/tools/parallelbuild.py @@ -72,14 +72,12 @@ def build_easyconfigs_in_parallel(build_command, easyconfigs, job_server = job_backend() if job_server is None: - _log.error("Cannot use --job if no job backend is available.") + raise EasyBuildError("Cannot use --job if no job backend is available.") try: job_server.begin() - except RuntimeError, err: - _log.error("connection to server failed (%s: %s), can't submit jobs." - % (err.__class__.__name__, err)) - return None # XXX: should this `raise` instead? + except RuntimeError as err: + raise EasyBuildError("connection to server failed (%s: %s), can't submit jobs.", err.__class__.__name__, err) # dependencies have already been resolved, # so one can linearly walk over the list and use previous job id's diff --git a/test/framework/parallelbuild.py b/test/framework/parallelbuild.py index 76d9b90715..a910b763bc 100644 --- a/test/framework/parallelbuild.py +++ b/test/framework/parallelbuild.py @@ -33,7 +33,9 @@ from vsc.utils.fancylogger import setLogLevelDebug, logToScreen from easybuild.framework.easyconfig.tools import process_easyconfig -from easybuild.tools import config, parallelbuild +from easybuild.tools import config, job +from easybuild.tools.job import pbs_python +from easybuild.tools.job.pbs_python import PbsPython from easybuild.tools.parallelbuild import build_easyconfigs_in_parallel from easybuild.tools.robot import resolve_dependencies @@ -59,37 +61,49 @@ def cleanup(self, *args, **kwargs): def has_holds(self, *args, **kwargs): pass - def submit(self, *args, **kwargs): + def _submit(self, *args, **kwargs): pass class ParallelBuildTest(EnhancedTestCase): """ Testcase for run module """ - def setUp(self): - """Set up testcase.""" - super(ParallelBuildTest, self).setUp() + def test_build_easyconfigs_in_parallel_pbs_python(self): + """Basic test for build_easyconfigs_in_parallel function.""" + # put mocked functions in place + PbsPython__init__ = PbsPython.__init__ + PbsPython_commit = PbsPython.commit + PbsPython_connect_to_server = PbsPython.connect_to_server + PbsPython_ppn = PbsPython.ppn + pbs_python_PbsJob = pbs_python.PbsJob + + PbsPython.__init__ = lambda self: PbsPython._init(self, pbs_server='localhost') + PbsPython.commit = mock + PbsPython.connect_to_server = mock + PbsPython.ppn = mock + pbs_python.PbsJob = MockPbsJob + build_options = { 'robot_path': os.path.join(os.path.dirname(__file__), 'easyconfigs'), 'valid_module_classes': config.module_classes(), 'validate': False, } - init_config(build_options=build_options) + init_config(args=['--job-backend=PbsPython'], build_options=build_options) - # put mocked functions in place - parallelbuild.connect_to_server = mock - parallelbuild.disconnect_from_server = mock - parallelbuild.get_ppn = mock - parallelbuild.PbsJob = MockPbsJob - def test_build_easyconfigs_in_parallel(self): - """Basic test for build_easyconfigs_in_parallel function.""" easyconfig_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.5-goolf-1.4.10.eb') easyconfigs = process_easyconfig(easyconfig_file) ordered_ecs = resolve_dependencies(easyconfigs) jobs = build_easyconfigs_in_parallel("echo %(spec)s", ordered_ecs, prepare_first=False) self.assertEqual(len(jobs), 8) + # restore mocked stuff + PbsPython.__init__ = PbsPython__init__ + PbsPython.commit = PbsPython_commit + PbsPython.connect_to_server = PbsPython_connect_to_server + PbsPython.ppn = PbsPython_ppn + pbs_python.PbsJob = pbs_python_PbsJob + def suite(): """ returns all the testcases in this module """ return TestLoader().loadTestsFromTestCase(ParallelBuildTest)