Fix #84: Improve import and initialization

Altered job, queue, and __init__ modules to delay any initialization that requires queue parsing until absolutely required—so all checks happen just in time, and not at import or initialization. This makes everything a lot faster and more efficient. Also fixes documentation on readthedocs and makes test skipping more robust in a variety of situations.
MikeDacre · Mar 13, 2018 · 499877d · 499877d
1 parent c948465
commit 499877d
Show file tree

Hide file tree

Showing 15 changed files with 241 additions and 127 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,9 @@
 # Ipython
 .ipynb_checkpoints
 
+# Vim
+.*neomake*
+
 # Apple's junk
 .DS_Store
 

diff --git a/docs/doc_requirements.txt b/docs/doc_requirements.txt
@@ -1,11 +1,9 @@
 # Requirements for building the Fyrd documentation
-pytest
-sphinx>=1.4.1
-CommonMark>=0.5.6
+Sphinx>=1.7.1
 dill>=0.2.5
-tabulate>=0.7.7
-six>=1.10.0
-tblib>=1.3.0
-sphinxcontrib-napoleon>=0.5.3
-versioneer>=0.18
-# sphinx-argparse>=0.2.0
+psutil>=5.2
+Pyro4>=4.70
+six>=1.11.0
+tqdm>=4.19.0
+tabulate>=0.8.2
+sqlalchemy>=1.2.0
diff --git a/docs/environment.yml b/docs/environment.yml
@@ -0,0 +1,13 @@
+name: fyrd-docs
+channels:
+    - conda-forge
+dependencies:
+    - python=3.6
+    - sphinx
+    - psutil
+    - Pyro4
+    - sqlalchemy
+    - dill
+    - tabulate
+    - pip:
+        - tqdm
diff --git a/docs/fyrd_manual.pdf b/docs/fyrd_manual.pdf
diff --git a/docs/sphinx/conf.py b/docs/sphinx/conf.py
@@ -1,22 +1,34 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-import os
+"""
+Instructions to build Fyrd Sphinx documentation.
+"""
 import sys
+import os
+from os import path as pth
+from re import match as rematch
+
+import six
+
+DIR = pth.abspath(os.curdir)
+sys.path.insert(0, pth.dirname(pth.dirname(DIR)))
+
 import fyrd
 
 # -- General configuration ------------------------------------------------
 
 project   = 'Fyrd'
-copyright = '2017, Michael Dacre <mike.dacre@gmail.com>'
+copyright = '2018, Michael Dacre <mike.dacre@gmail.com>'
 author    = 'Michael Dacre <mike.dacre@gmail.com>'
-version   = '0.6'
-release   = fyrd.version
-#  release   = '0.6.2b1'
+version   = rematch('\d+\.\d+\.\d+', fyrd.__version__).group()
+release   = fyrd.__version__
 language  = 'en'
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
+on_rtd = os.environ.get('READTHEDOCS') == 'True'
+
 if 'latex' in sys.argv:
     master_doc = 'simple_index'
 else:
@@ -27,15 +39,31 @@
 extensions = [
     'sphinx.ext.autodoc',
     'sphinx.ext.napoleon',
+    'sphinx.ext.intersphinx',
     'sphinx.ext.doctest',
     'sphinx.ext.todo',
     'sphinx.ext.coverage',
     'sphinx.ext.ifconfig',
     'sphinx.ext.viewcode',
 ]
 
+napoleon_use_admonition_for_examples = True
+
+intersphinx_mapping = {
+    'sphinx': ('http://www.sphinx-doc.org/en/master/', None),
+    'Pyro4': ('https://pythonhosted.org/Pyro4/', None),
+    'SQLAlchemy': ('http://docs.sqlalchemy.org/en/latest/', None),
+}
+if six.PY3:
+    intersphinx_mapping['python'] = ('https://docs.python.org/3.6/', None)
+else:
+    intersphinx_mapping['python'] = ('https://docs.python.org/2.7/', None)
+
+
 # Autodoc configuration
+autodoc_default_flags = ['show_inheritance', 'autosummary']
 autoclass_content = 'both'
+autodata_content = 'call'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -76,12 +104,8 @@
 
 # -- Options for HTML output ----------------------------------------------
 
-on_rtd = os.environ.get('READTHEDOCS') == 'True'
-if on_rtd:
-    html_theme = 'default'
-else:
+if not on_rtd:
     html_theme = 'alabaster'
-    #  html_theme = 'nature'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the

diff --git a/fyrd/__init__.py b/fyrd/__init__.py
@@ -141,18 +141,14 @@
 #  Version  #
 #############
 
-# Planned version
-version = '0.6.2a1'
 from ._version import get_versions
-# Actual version (same as planned for releases)
 __version__ = get_versions()['version']
 del get_versions
 
 ###################
 #  House Keeping  #
 ###################
 
-
 class ClusterError(Exception):
 
     """A custom exception for cluster errors."""
@@ -164,7 +160,6 @@ class ClusterError(Exception):
 #  Make our functions easily available  #
 #########################################
 
-
 from . import queue
 from . import job
 from . import helpers
@@ -199,9 +194,3 @@ class ClusterError(Exception):
 __all__ = ['Job', 'Queue', 'wait', 'get', 'submit', 'submit_file', 'jobify',
            'make_job_file', 'clean', 'clean_dir', 'check_queue', 'option_help',
            'set_profile', 'get_profile', 'get_profiles', 'conf', 'helpers']
-
-##########################
-#  Set the cluster type  #
-##########################
-
-batch_systems.MODE = get_cluster_environment()
diff --git a/fyrd/basic.py b/fyrd/basic.py
@@ -50,27 +50,29 @@
 
 
 def submit(command, args=None, kwargs=None, name=None, qtype=None,
-           profile=None, **kwds):
+           profile=None, queue=None, **kwds):
     """Submit a script to the cluster.
 
     Parameters
     ----------
     command : function/str
         The command or function to execute.
-    args : tuple/dict
+    args : tuple/dict, optional
         Optional arguments to add to command, particularly useful for
         functions.
-    kwargs : dict
+    kwargs : dict, optional
         Optional keyword arguments to pass to the command, only used for
         functions.
-    name : str
-        Optional name of the job. If not defined, guessed. If a job of the same
-        name is already queued, an integer job number (not the queue number)
-        will be added, ie.  <name>.1
-    qtype : str
+    name : str, optional
+        Optional name of the job. If not defined, guessed. If a job of the
+        same name is already queued, an integer job number (not the queue
+        number) will be added, ie.  <name>.1
+    qtype : str, optional
         Override the default queue type
-    profile : str
+    profile : str, optional
         The name of a profile saved in the conf
+    queue : fyrd.queue.Queue, optional
+        An already initiated Queue class to use.
     kwds
         *All other keywords are parsed into cluster keywords by the options
         system.* For available keywords see `fyrd.option_help()`
@@ -83,7 +85,7 @@ def submit(command, args=None, kwargs=None, name=None, qtype=None,
     _batch.check_queue()  # Make sure the queue.MODE is usable
 
     job = Job(command=command, args=args, kwargs=kwargs, name=name,
-              qtype=qtype, profile=profile, **kwds)
+              qtype=qtype, profile=profile, queue=queue, **kwds)
 
     job.write()
     job.submit()
@@ -98,27 +100,29 @@ def submit(command, args=None, kwargs=None, name=None, qtype=None,
 
 
 def make_job(command, args=None, kwargs=None, name=None, qtype=None,
-             profile=None, **kwds):
+             profile=None, queue=None, **kwds):
     """Make a job compatible with the chosen cluster but do not submit.
 
     Parameters
     ----------
     command : function/str
         The command or function to execute.
-    args : tuple/dict
+    args : tuple/dict, optional
         Optional arguments to add to command, particularly useful for
         functions.
-    kwargs : dict
+    kwargs : dict, optional
         Optional keyword arguments to pass to the command, only used for
         functions.
-    name : str
-        Optional name of the job. If not defined, guessed. If a job of the same
-        name is already queued, an integer job number (not the queue number)
-        will be added, ie.  <name>.1
-    qtype : str
+    name : str, optional
+        Optional name of the job. If not defined, guessed. If a job of the
+        same name is already queued, an integer job number (not the queue
+        number) will be added, ie.  <name>.1
+    qtype : str, optional
         Override the default queue type
-    profile : str
+    profile : str, optional
         The name of a profile saved in the conf
+    queue : fyrd.queue.Queue, optional
+        An already initiated Queue class to use.
     kwds
         *All other keywords are parsed into cluster keywords by the options
         system.* For available keywords see `fyrd.option_help()`
@@ -131,34 +135,36 @@ def make_job(command, args=None, kwargs=None, name=None, qtype=None,
     _batch.check_queue()  # Make sure the queue.MODE is usable
 
     job = Job(command=command, args=args, kwargs=kwargs, name=name,
-              qtype=qtype, profile=profile, **kwds)
+              qtype=qtype, profile=profile, queue=queue, **kwds)
 
     # Return the path to the script
     return job
 
 
 def make_job_file(command, args=None, kwargs=None, name=None, qtype=None,
-                  profile=None, **kwds):
+                  profile=None, queue=None, **kwds):
     """Make a job file compatible with the chosen cluster.
 
     Parameters
     ----------
     command : function/str
         The command or function to execute.
-    args : tuple/dict
+    args : tuple/dict, optional
         Optional arguments to add to command, particularly useful for
         functions.
-    kwargs : dict
+    kwargs : dict, optional
         Optional keyword arguments to pass to the command, only used for
         functions.
-    name : str
-        Optional name of the job. If not defined, guessed. If a job of the same
-        name is already queued, an integer job number (not the queue number)
-        will be added, ie.  <name>.1
-    qtype : str
+    name : str, optional
+        Optional name of the job. If not defined, guessed. If a job of the
+        same name is already queued, an integer job number (not the queue
+        number) will be added, ie.  <name>.1
+    qtype : str, optional
         Override the default queue type
-    profile : str
+    profile : str, optional
         The name of a profile saved in the conf
+    queue : fyrd.queue.Queue, optional
+        An already initiated Queue class to use.
     kwds
         *All other keywords are parsed into cluster keywords by the options
         system.* For available keywords see `fyrd.option_help()`
@@ -172,7 +178,7 @@ def make_job_file(command, args=None, kwargs=None, name=None, qtype=None,
     _batch.check_queue()  # Make sure the queue.MODE is usable
 
     job = Job(command=command, args=args, kwargs=kwargs, name=name,
-              qtype=qtype, profile=profile, **kwds)
+              qtype=qtype, profile=profile, queue=queue, **kwds)
 
     job = job.write()
 
@@ -376,9 +382,13 @@ def clean_dir(directory=None, suffix=None, qtype=None, confirm=False,
 ###############################################################################
 
 
-def wait(jobs, notify=True):
+def wait(jobs, notify=True, queue=None):
     """Wait for jobs to finish.
 
+    Only works on user jobs by default. To work on jobs so someone else,
+    initialize a fyrd.queue.Queue class with their user info and pass as an
+    argument to queue.
+
     Parameters
     ----------
     jobs : fyrd.job.Job or str or list of either (mixed list fine)
@@ -390,19 +400,25 @@ def wait(jobs, notify=True):
         If a string is passed, notification is forced and the string must
         be the to address.
         False means no notification
+    queue : fyrd.queue.Queue, optional
+        An already initiated Queue class to use.
 
     Returns
     -------
     success : bool
         True if all jobs successful, false otherwise
     """
-    q = _queue.Queue()
+    q = queue if queue else _queue.default_queue()
     return q.wait(jobs, notify=notify)
 
 
-def get(jobs):
+def get(jobs, queue=None):
     """Get results of jobs when they complete.
 
+    Only works on user jobs by default. To work on jobs so someone else,
+    initialize a fyrd.queue.Queue class with their user info and pass as an
+    argument to queue.
+
     Parameters
     ----------
     jobs : fyrd.job.Job or list of fyrd.job.Job
@@ -411,9 +427,11 @@ def get(jobs):
     -------
     list
         Outputs (STDOUT or return value) of jobs
+    queue : fyrd.queue.Queue, optional
+        An already initiated Queue class to use.
 
     .. note:: This function also modifies the input Job objects, so they will
               contain all outputs and state information.
     """
-    q = _queue.Queue()
+    q = queue if queue else _queue.default_queue()
     return q.get(jobs)