Merge pull request #4 from boegel/gc3pie

make unit test suite pass, rename JobServer to JobBackend, fix default for --job-bakcend

easybuild/tools/config.py

@@ -52,6 +52,7 @@
 _log = fancylogger.getLogger('config', fname=False)
 
 
+DEFAULT_JOB_BACKEND = 'PbsPython'
 DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log")
 DEFAULT_MNS = 'EasyBuildMNS'
 DEFAULT_MODULE_SYNTAX = 'Tcl'
@@ -68,8 +69,6 @@
 DEFAULT_REPOSITORY = 'FileRepository'
 DEFAULT_STRICT = run.WARN
 
-PREFERRED_JOB_BACKENDS = ('Pbs', 'GC3Pie')
-
 
 # utility function for obtaining default paths
 def mk_full_default_path(name, prefix=DEFAULT_PREFIX):

easybuild/tools/job/__init__.py

@@ -30,11 +30,10 @@
 from vsc.utils.missing import get_subclasses
 
 from easybuild.tools.config import get_job_backend
-from easybuild.tools.config import PREFERRED_JOB_BACKENDS
 from easybuild.tools.utilities import import_available_modules
 
 
-class JobServer(object):
+class JobBackend(object):
     __metaclass__ = ABCMeta
 
     USABLE = False
@@ -92,9 +91,7 @@ def avail_job_backends(check_usable=True):
     Return all known job execution backends.
     """
     import_available_modules('easybuild.tools.job')
-    class_dict = dict([(x.__name__, x)
-                       for x in get_subclasses(JobServer)
-                       if (x.USABLE or not check_usable)])
+    class_dict = dict([(x.__name__, x) for x in get_subclasses(JobBackend)])
     return class_dict
 
 
@@ -107,16 +104,3 @@ def job_backend():
         return None
     job_backend_class = avail_job_backends().get(job_backend)
     return job_backend_class()
-
-
-def preferred_job_backend(order=PREFERRED_JOB_BACKENDS):
-    """
-    Return name of preferred concrete `JobServer` instance, or `None`
-    if none is available.
-    """
-    available_backends = avail_job_backends()
-    for backend in order:
-        if backend in available_backends:
-            return backend
-            break
-    return None

easybuild/tools/job/gc3pie.py

@@ -44,7 +44,7 @@
     HAVE_GC3PIE = False
 
 from easybuild.tools.build_log import print_msg
-from easybuild.tools.job import JobServer
+from easybuild.tools.job import JobBackend
 
 from vsc.utils import fancylogger
 
@@ -56,7 +56,7 @@
 
 
 # eb --job --job-backend=GC3Pie
-class GC3Pie(JobServer):
+class GC3Pie(JobBackend):
     """
     Use the GC3Pie__ framework to submit and monitor compilation jobs.
 
@@ -83,7 +83,7 @@ def make_job(self, script, name, env_vars=None, hours=None, cores=None):
         Create and return a job object with the given parameters.
 
         First argument `server` is an instance of the corresponding
-        `JobServer` class, i.e., a `GC3Pie`:class: instance in this case.
+        `JobBackend` class, i.e., a `GC3Pie`:class: instance in this case.
 
         Second argument `script` is the content of the job script
         itself, i.e., the sequence of shell commands that will be

easybuild/tools/job/pbs_python.py

@@ -36,7 +36,7 @@
 from vsc.utils import fancylogger
 
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.job import JobServer
+from easybuild.tools.job import JobBackend
 
 
 _log = fancylogger.getLogger('pbs_python', fname=False)
@@ -58,27 +58,30 @@ def only_if_pbs_import_successful(fn):
     HAVE_PBS_PYTHON = True
 except ImportError:
     _log.debug("Failed to import pbs from pbs_python."
-               " Silently ignoring, this is a real issue only with --job=pbs")
+               " Silently ignoring, this is a real issue only when pbs_python is used as backend for --job")
     # no `pbs_python` available, turn function into a no-op
     def only_if_pbs_import_successful(fn):
         def instead(*args, **kwargs):
             """This is a no-op since `pbs_python` is not available."""
             errmsg = ("PBSQuery or pbs modules not available."
                       " Please make sure `pbs_python` is installed and usable.")
-            _log.error(errmsg)
-            raise RuntimeError(errmsg)
+            raise EasyBuildError(errmsg)
         return instead
     HAVE_PBS_PYTHON = False
 
 
-class Pbs(JobServer):
+class PbsPython(JobBackend):
     """
     Manage PBS server communication and create `PbsJob` objects.
     """
 
     USABLE = HAVE_PBS_PYTHON
 
+    @only_if_pbs_import_successful
     def __init__(self, pbs_server=None):
+        _init()
+
+    def _init(self, pbs_server=None):
         self.pbs_server = pbs_server or pbs.pbs_default()
         self.conn = None
         self._ppn = None

easybuild/tools/options.py

@@ -48,16 +48,15 @@
 from easybuild.framework.easyconfig.templates import template_documentation
 from easybuild.framework.easyconfig.tools import get_paths_for
 from easybuild.framework.extension import Extension
-from easybuild.tools import build_log, config, run  # @UnusedImport make sure config is always initialized!
+from easybuild.tools import build_log, config, run  # build_log should always stay there, to ensure EasyBuildLog
 from easybuild.tools.build_log import EasyBuildError, raise_easybuilderror
-from easybuild.tools.config import DEFAULT_LOGFILE_FORMAT, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL
-from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PREFIX, DEFAULT_REPOSITORY
-from easybuild.tools.config import DEFAULT_STRICT, get_pretend_installpath, mk_full_default_path
-from easybuild.tools.config import PREFERRED_JOB_BACKENDS
+from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX
+from easybuild.tools.config import DEFAULT_MODULES_TOOL, DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PREFIX
+from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_STRICT, get_pretend_installpath, mk_full_default_path
 from easybuild.tools.configobj import ConfigObj, ConfigObjError
 from easybuild.tools.docs import FORMAT_RST, FORMAT_TXT, avail_easyconfig_params
 from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, fetch_github_token
-from easybuild.tools.job import avail_job_backends, preferred_job_backend
+from easybuild.tools.job import avail_job_backends
 from easybuild.tools.modules import avail_modules_tools
 from easybuild.tools.module_generator import ModuleGeneratorLua, avail_module_generators
 from easybuild.tools.module_naming_scheme import GENERAL_CLASS
@@ -249,7 +248,7 @@ def config_options(self):
             'installpath-software': ("Install path for software (if None, combine --installpath and --subdir-software)",
                                      None, 'store', None),
             'job-backend': ("What job runner to use", 'choice', 'store',
-                            preferred_job_backend(), (avail_job_backends().keys())),
+                            DEFAULT_JOB_BACKEND, avail_job_backends().keys()),
             # purposely take a copy for the default logfile format
             'logfile-format': ("Directory name and format of the log file",
                                'strtuple', 'store', DEFAULT_LOGFILE_FORMAT[:], {'metavar': 'DIR,FORMAT'}),

easybuild/tools/parallelbuild.py

@@ -72,14 +72,12 @@ def build_easyconfigs_in_parallel(build_command, easyconfigs,
 
     job_server = job_backend()
     if job_server is None:
-        _log.error("Cannot use --job if no job backend is available.")
+        raise EasyBuildError("Cannot use --job if no job backend is available.")
 
     try:
         job_server.begin()
-    except RuntimeError, err:
-        _log.error("connection to server failed (%s: %s), can't submit jobs."
-                   % (err.__class__.__name__, err))
-        return None # XXX: should this `raise` instead?
+    except RuntimeError as err:
+        raise EasyBuildError("connection to server failed (%s: %s), can't submit jobs.", err.__class__.__name__, err)
 
     # dependencies have already been resolved,
     # so one can linearly walk over the list and use previous job id's

test/framework/parallelbuild.py

@@ -33,7 +33,9 @@
 from vsc.utils.fancylogger import setLogLevelDebug, logToScreen
 
 from easybuild.framework.easyconfig.tools import process_easyconfig
-from easybuild.tools import config, parallelbuild
+from easybuild.tools import config, job
+from easybuild.tools.job import pbs_python
+from easybuild.tools.job.pbs_python import PbsPython
 from easybuild.tools.parallelbuild import build_easyconfigs_in_parallel
 from easybuild.tools.robot import resolve_dependencies
 
@@ -59,37 +61,49 @@ def cleanup(self, *args, **kwargs):
     def has_holds(self, *args, **kwargs):
         pass
 
-    def submit(self, *args, **kwargs):
+    def _submit(self, *args, **kwargs):
         pass
 
 
 class ParallelBuildTest(EnhancedTestCase):
     """ Testcase for run module """
 
-    def setUp(self):
-        """Set up testcase."""
-        super(ParallelBuildTest, self).setUp()
+    def test_build_easyconfigs_in_parallel_pbs_python(self):
+        """Basic test for build_easyconfigs_in_parallel function."""
+        # put mocked functions in place
+        PbsPython__init__ = PbsPython.__init__
+        PbsPython_commit = PbsPython.commit
+        PbsPython_connect_to_server = PbsPython.connect_to_server
+        PbsPython_ppn = PbsPython.ppn
+        pbs_python_PbsJob = pbs_python.PbsJob
+
+        PbsPython.__init__ = lambda self: PbsPython._init(self, pbs_server='localhost')
+        PbsPython.commit = mock
+        PbsPython.connect_to_server = mock
+        PbsPython.ppn = mock
+        pbs_python.PbsJob = MockPbsJob
+
         build_options = {
             'robot_path': os.path.join(os.path.dirname(__file__), 'easyconfigs'),
             'valid_module_classes': config.module_classes(),
             'validate': False,
         }
-        init_config(build_options=build_options)
+        init_config(args=['--job-backend=PbsPython'], build_options=build_options)
 
-        # put mocked functions in place
-        parallelbuild.connect_to_server = mock
-        parallelbuild.disconnect_from_server = mock
-        parallelbuild.get_ppn = mock
-        parallelbuild.PbsJob = MockPbsJob
 
-    def test_build_easyconfigs_in_parallel(self):
-        """Basic test for build_easyconfigs_in_parallel function."""
         easyconfig_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.5-goolf-1.4.10.eb')
         easyconfigs = process_easyconfig(easyconfig_file)
         ordered_ecs = resolve_dependencies(easyconfigs)
         jobs = build_easyconfigs_in_parallel("echo %(spec)s", ordered_ecs, prepare_first=False)
         self.assertEqual(len(jobs), 8)
 
+        # restore mocked stuff
+        PbsPython.__init__ = PbsPython__init__
+        PbsPython.commit = PbsPython_commit
+        PbsPython.connect_to_server = PbsPython_connect_to_server
+        PbsPython.ppn = PbsPython_ppn
+        pbs_python.PbsJob = pbs_python_PbsJob
+
 def suite():
     """ returns all the testcases in this module """
     return TestLoader().loadTestsFromTestCase(ParallelBuildTest)