notebook UI version 2

docs/dev_notes.rst

@@ -0,0 +1,13 @@
+.. _dev_notes:
+
+Dev notes
+---------
+
+Logging
+"""""""
+
+Use python logging module.
+
+See test/imars3d/imars3d.conf for example of logging configuration.
+
+Example of logging code: imars3d.decorators.mpi_parallelize.

notebooks/TomoRecon-UI_v2.ipynb

@@ -0,0 +1,139 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Quick Interface Instructions:\n",
+    "* Make a copy by clicking \"File -> Make a copy\"\n",
+    "* Give the new notebook a good name\n",
+    "* Before starting, make sure the circle indicating the kernel status on the top-right corner is open (meaning the jupyter kernel is ready)\n",
+    "* Run the cells below sequentially\n",
+    "* __Shift-Enter__ to run a cell\n",
+    "* When a cell's number is '\\*', the kernel is busy calculating for you. Please wait.\n",
+    "* Click a cell to edit its contents"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Import useful tools\n",
+    "__Be patient, this will take a little while.__"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "from imars3d.jnbui.tomoreconui import createContext, wizard"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Wizard\n",
+    "For the configuration steps, the following inputs work:\n",
+    "* IPTS Number: 15518\n",
+    "* Scan Name: derek_inj\n",
+    "* CT Signature: Derek_injec\n",
+    "* DF/OB Files: all files under df/ob directory\n",
+    "\n",
+    "When you reach the step with the tabs, you __must__ zoom into your desired Region of Interest. The Image Slider's ROI field displays the ROI that will be used for reconstruction."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "context = createContext()\n",
+    "wizard(context)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Run this cell to see the different options for the wizard function."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "help(wizard)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Examining Intermediate Output\n",
+    "You can examine any output of the wizard (such as sinograms) by replacing \"sinograms\" in the final cell with any of the options listed by running the next cell."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dir(context.ct.r)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from ipywe import imageslider\n",
+    "is1 = imageslider.ImageSlider(context.ct.r.sinograms, 300, 300)\n",
+    "is1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 2",
+   "language": "python",
+   "name": "python2"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notebooks/imars3d.conf

@@ -0,0 +1,32 @@
+parallelization:
+  max_nodes: 20
+
+logging:
+  version: 1
+  formatters:
+    simple:
+      format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+  handlers:
+    mainlog:
+      class: logging.FileHandler
+      level: DEBUG
+      formatter: simple
+      filename: imars3d.log
+  loggers:
+    imars3d.tilt:
+      level: DEBUG
+      handlers: [mainlog]
+      propagate: no
+    mpi:
+      level: DEBUG
+      handlers: [mainlog]
+      propagate: no
+  root:
+    level: DEBUG
+    handlers: [mainlog]
+
+progress_bar:
+  term_width: 70
+
+CT:
+  clean_intermediate_files: false

python/imars3d/CT.py

@@ -6,6 +6,10 @@
 import imars3d as i3
 import progressbar
 from . import decorators as dec
+from imars3d import configuration
+pb_config = configuration['progress_bar']
+ct_config = configuration.get('CT', dict(clean_intermediate_files="archive"))
+
 
 class CT:
 
@@ -48,21 +52,23 @@ class CT:
 
 or they can be kept where it is:
 
-    clean_intermediate_files=None
+    clean_intermediate_files=False
 
 and you will need to clean them up yourself.
+The default behavior can be modified by configuration file "imars3d.conf".
 """
 
     def __init__(
             self, path, CT_subdir=None, CT_identifier=None,
             workdir='work', outdir='out', 
             parallel_preprocessing=True, parallel_nodes=None,
-            clean_intermediate_files='archive',
+            clean_intermediate_files=None,
             vertical_range=None,
             ob_identifier=None, df_identifier=None,
             ob_files=None, df_files=None,
             skip_df=False,
     ):
+        import logging; self.logger = logging.getLogger("CT")
         self.path = path
         if CT_subdir is not None:
             # if ct is in a subdir, its name most likely the
@@ -106,7 +112,9 @@ def __init__(
 
         self.parallel_preprocessing = parallel_preprocessing
         self.parallel_nodes = parallel_nodes
-        assert clean_intermediate_files in ['on_the_fly', 'archive', None]
+        if clean_intermediate_files is None:
+            clean_intermediate_files = ct_config['clean_intermediate_files']
+        assert clean_intermediate_files in ['on_the_fly', 'archive', False]
         self.clean_intermediate_files = clean_intermediate_files
         self.vertical_range = vertical_range
         self.r = results()
@@ -268,7 +276,11 @@ def recon(self,
         if self.clean_intermediate_files == 'archive':
             import datetime
             now = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
-            shutil.move(workdir, os.path.join(outdir, 'work-%s' % now))
+            newpath = os.path.join(outdir, 'work-%s' % now)
+            shutil.move(workdir, newpath)
+            # create a soft link so that the intermediate data can still be accessed
+            # from the CT object
+            os.symlink(newpath, workdir)
         elif self.clean_intermediate_files == 'on_the_fly':
             shutil.rmtree(workdir)
         return
@@ -594,14 +606,16 @@ def fn2angle(fn):
                     progressbar.Bar(),
                     ' [', progressbar.ETA(), '] ',
                 ],
-                max_value = len(angles) - 1
+                max_value = len(angles) - 1,
+                **pb_config
             )
             for i, angle in enumerate(angles):
                 try:
                     ifs.getFilename(angle)
                 except:
                     import traceback as tb
-                    tb.print_exc()
+                    m = tb.format_exc()
+                    self.logger.debug("i=%s,angle=%s: %s" % (i, angle, m))
                     bad = True
                     break
                 bar.update(i)

python/imars3d/__init__.py

@@ -4,11 +4,9 @@
 import matplotlib as mpl; mpl.use("Agg")
 
 
-
 # config file
 # Examples
-#   * tests/imars3d/tilt/imars3d.conf
-#   * tests/imars3d/parallel/imars3d.conf
+#   * tests/imars3d/imars3d.conf
 import yaml, os
 conf_path = "imars3d.conf"
 config = dict()
@@ -19,10 +17,14 @@
 logging_conf = config.get("logging")
 if logging_conf:
     logging.config.dictConfig(logging_conf)
-# parallelization cofnig
+# parallelization config
 parallel = config.get('parallelization')
 if not parallel:
     config['parallelization'] = dict(max_nodes=20)
+# progressbar config
+progress_bar = config.get('progress_bar')
+if not progress_bar:
+    config['progress_bar'] = dict(term_width=80)
 # have to rename. config is a subpackage
 configuration = config
 

python/imars3d/components/projection.py

@@ -3,10 +3,11 @@
 
 
 import progressbar
-
-from .AbstractComponent import AbstractComponent
+from imars3d import configuration
+pb_config = configuration['progress_bar']
 
 
+from .AbstractComponent import AbstractComponent
 class Projection(AbstractComponent):
 
     def __call__(self, ct_series, sinograms):
@@ -51,7 +52,8 @@ def __call__(self, ct_series, sinograms):
         progressbar.Bar(),
         ' [', progressbar.ETA(), '] ',
     ],
-    max_value = max
+    max_value = max,
+    **pb_config
 )
 
 
@@ -63,6 +65,7 @@ def __init__(self, num_workers=None):
 
     def __call__(self, ct_series, sinograms):
         """convert ct image series to sinogram series"""
+        print ("* Projecting...")
         # get metadata for sinograms
         data0 = ct_series[0].data
         Y, X = data0.shape
@@ -82,9 +85,9 @@ def __call__(self, ct_series, sinograms):
         script = os.path.join(os.path.dirname(__file__), 'projection_mp.py')
         cmd = 'python %s %s' % (script, args_pkl)
         # 
-        from ..shutils import exec_withlog
-        logfile = os.path.join(dir, 'log.run')
-        exec_withlog(cmd, logfile)
+        import subprocess as sp, shlex
+        args = shlex.split(cmd)
+        sp.check_call(args, shell=False)
         return
 
 

python/imars3d/components/projection_mp.py

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 
 import multiprocessing as mp, numpy as np, pickle, os
+import logging; logger = logging.getLogger("projection")
 
 # global array
 data = None
@@ -39,7 +40,7 @@ def job_handler(y):
     sinograms = pickle.loads(sinograms_pkl)
     sino = sinograms[y]
     if sinograms.exists(y):
-        print("%s already existed" % sino)
+        logger.debug("%s already existed" % sino)
         return y
     sino.data = data[:, y, :]
     from imars3d.filters.smoothing import filter_one_median as smooth

python/imars3d/decorators.py

@@ -30,6 +30,7 @@ def mpi_parallelize(f):
 """ + import_statement + """
 method(*args, **kwds)
 """
+    import logging; logger = logging.getLogger("mpi")
     tmpdir="_mpi_tmp/%s" % (f.__name__,)
     if not os.path.exists(tmpdir):
         os.makedirs(tmpdir)
@@ -55,23 +56,14 @@ def _(*args, **kwds):
         nodes = min(nodes, imars3d.configuration['parallelization']['max_nodes'])
         # shell cmd
         cmd = 'mpirun -np %(nodes)s python %(pyfile)s' % locals()
-        print("* running %s" % cmd)
-        print("  - args: %s" % (args,))
-        print("  - kwds:")
+        logger.info("* running %s" % cmd)
+        logger.info("  - args: %s" % (args,))
+        logger.info("  - kwds:")
         for k,v in kwds.items():
-            print("    - %s: %s" % (k,v))
+            logger.info("    - %s: %s" % (k,v))
             continue
-        import subprocess as sp, shlex
-        args = shlex.split(cmd)
-        logfile = os.path.join(dir, 'log.run')
-        print("  - logging in %s" % os.path.abspath(logfile))
-        outstream = open(logfile, 'wt')
-        outstream.write('%s\n\n' % cmd)
-        if sp.call(args, stdout=outstream, stderr=outstream, shell=False):
-            outstream.close()
-            text = open(logfile).read()
-            raise RuntimeError("%s failed.\n%s\n" % (cmd, text))
-            # raise RuntimeError("%s failed. See log file %s" % (cmd, logfile))
-        print("done.")
+        from .shutils import exec_redirect_to_stdout
+        exec_redirect_to_stdout(cmd)
+        logger.info("done.")
         return
     return _