Fix morphval

Change-Id: I3216edc1165254ba52114306df53f2753eeda5b9

morphval/__init__.py

@@ -1,4 +1,4 @@
-'''Morphology Statistical Validation Package'''
+"""Morphology Statistical Validation Package"""
 import pkg_resources
 
 __version__ = pkg_resources.get_distribution("synthesis_workflow").version

morphval/cli.py

@@ -1,36 +1,52 @@
 #!/usr/bin/env python
+"""CLI for MorphVal package"""
 import argparse
 
-# TODO: remove this once NeuroM properly handles figures:
-# https://github.com/BlueBrain/NeuroM/issues/590
-import matplotlib
-matplotlib.use('Agg')
-
 from morphval.validation_main import Validation
 from morphval import config
 
 
 def get_parser():
-    '''return the argument parser'''
+    """return the argument parser"""
     parser = argparse.ArgumentParser()
-    parser.add_argument('-t', '--test-dir', required=True,
-                        help='full path to directory with test data')
-    parser.add_argument('-r', '--ref-dir', required=True,
-                        help='full path to directory with reference data')
-    parser.add_argument('-o', '--output-dir', required=True,
-                        help='full path to directory for the validation results')
-    parser.add_argument('-c', '--config', required=True,
-                        help='full path to yaml config file')
-    parser.add_argument('--example-config', action='store_true',
-                        help='print out an example config')
-    parser.add_argument('--bio-compare', action='store_true', default=False,
-                        help='Use the bio compare template')
-    parser.add_argument('--cell-figure-count', default=10, type=int,
-                        help='Number of example cells to show')
+    parser.add_argument(
+        "-t", "--test-dir", required=True, help="full path to directory with test data"
+    )
+    parser.add_argument(
+        "-r",
+        "--ref-dir",
+        required=True,
+        help="full path to directory with reference data",
+    )
+    parser.add_argument(
+        "-o",
+        "--output-dir",
+        required=True,
+        help="full path to directory for the validation results",
+    )
+    parser.add_argument(
+        "-c", "--config", required=True, help="full path to yaml config file"
+    )
+    parser.add_argument(
+        "--example-config", action="store_true", help="print out an example config"
+    )
+    parser.add_argument(
+        "--bio-compare",
+        action="store_true",
+        default=False,
+        help="Use the bio compare template",
+    )
+    parser.add_argument(
+        "--cell-figure-count",
+        default=10,
+        type=int,
+        help="Number of example cells to show",
+    )
     return parser
 
 
 def main():
+    """Main function of MorphVal package"""
     args = get_parser().parse_args()
 
     if args.example_config:
@@ -43,5 +59,5 @@ def main():
     validation.write_report(validation_report=(not args.bio_compare))
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

morphval/common.py

@@ -1,14 +1,14 @@
-'''common.py - private helper functions of the validation module'''
+"""common.py - private helper functions of the validation module"""
 import contextlib
 import os
 import json
 from copy import deepcopy
 
-import neurom
 from neurom import viewer, geom, NeuriteType
 from neurom.core import Population, iter_neurites
 from neurom.core.dataformat import COLS
 from neurom.core.types import tree_type_checker as is_type
+from region_grower.utils import NumpyEncoder
 
 import numpy as np
 
@@ -18,53 +18,55 @@
 
 
 COMP_MAP = {
-    'basal_dendrite': NeuriteType.basal_dendrite,
-    'apical_dendrite': NeuriteType.apical_dendrite,
-    'axon': NeuriteType.axon,
-    'soma': None,
+    "basal_dendrite": NeuriteType.basal_dendrite,
+    "apical_dendrite": NeuriteType.apical_dendrite,
+    "axon": NeuriteType.axon,
+    "soma": None,
 }
 
 
 def pretty_name(name):
-    return name.replace('_', ' ')
+    """Make a pretty name"""
+    return name.replace("_", " ")
 
 
 def add_progress_bar(items, template, description):
+    """Add a progress bar in notebooks"""
     if description:
-        from tqdm import tqdm_notebook
+        from tqdm import tqdm_notebook  # pylint: disable=import-outside-toplevel
+
         return tqdm_notebook(items, desc=template.format(description))
     return items
 
 
 def progress_bar_label(template, description, notebook):
+    """Format progress bar label"""
     return template.format(description) if notebook else None
 
 
 def dump2json(data_dir, var_name, data):
-    ''' Saves the dictionary 'data' into a .json file.
+    """Saves the dictionary 'data' into a .json file.
     data_dir : the data directory
     var_name : the name of the dictionary as a string
     data : the data dictionary
-    '''
-    fname = os.path.join(data_dir, var_name + '.json')
-    with open(fname, 'w') as fd:
-        json.dump(data, fd, indent=2, sort_keys=True)
+    """
+    fname = os.path.join(data_dir, var_name + ".json")
+    with open(fname, "w") as fd:
+        json.dump(data, fd, cls=NumpyEncoder, indent=2, sort_keys=True)
     return fname
 
 
 def load_json(fname):
-    '''loads a json file from file with fname into the results dictionary
-    returns the results dictionary'''
-    with open(fname, 'r') as fd:
+    """loads a json file from file with fname into the results dictionary
+    returns the results dictionary"""
+    with open(fname, "r") as fd:
         return json.load(fd)
 
 
 def find_cells(dir_name):
-    '''Returns the cells in dir_name'''
+    """Returns the cells in dir_name"""
     # now we skip all hidden files, starting with '.'
-    files = [os.path.join(dir_name, f)
-             for f in os.listdir(dir_name)
-             if f[0] != '.']
+    files = [os.path.join(dir_name, f) for f in os.listdir(dir_name) if f[0] != "."]
 
     if not files:
         raise Exception("There are no cells in '" + dir_name + "'")
@@ -74,10 +76,10 @@ def find_cells(dir_name):
 
 @contextlib.contextmanager
 def pyplot_non_interactive():
-    '''suppress pyplot showing blank graphs when in interactive mode
+    """suppress pyplot showing blank graphs when in interactive mode
 
     This usually happens in the context of jupyter notebooks
-    '''
+    """
     if plt.isinteractive():
         plt.ioff()
         yield
@@ -88,80 +90,92 @@ def pyplot_non_interactive():
 
 @contextlib.contextmanager
 def get_agg_fig():
+    """Yield a figure and close it afterwards"""
     fig = Figure()
-    canvas = FigureCanvas(fig)
+    FigureCanvas(fig)
     yield fig
     plt.close(fig)
 
 
 def center_population(population):
-    '''returns a new population where all cells have been translated to their soma origins'''
-    morphs = [geom.transform.translate(n, -np.asarray(n.soma.center))
-              for n in population]
+    """returns a new population where all cells have been translated to their soma origins"""
+    morphs = [
+        geom.transform.translate(n, -np.asarray(n.soma.center)) for n in population
+    ]
     return Population(morphs, name=population.name)
 
 
 def truncate_population(population, count):
-    '''returns a new population with `count` elements'''
+    """returns a new population with `count` elements"""
     morphs = [population[i] for i in range(min(len(population), count))]
     return Population(morphs, name=population.name)
 
 
 def get_components_population(population, component):
-    if component == 'full_morph':
+    """Get component of a given population"""
+    if component == "full_morph":
         return population
 
     def filtered_neurites(n):
-        return [c for c in iter_neurites(n, filt=is_type(COMP_MAP[component]))]
+        return list(iter_neurites(n, filt=is_type(COMP_MAP[component])))
 
     morphs = []
     for n in population:
         nrn = deepcopy(n)
         nrn.neurites = filtered_neurites(n)
-        nrn.name = n.name + '_' + component
+        nrn.name = n.name + "_" + component
         morphs.append(nrn)
 
     return Population(morphs, name=population.name)
 
 
 def compute_bounding_box(*populations):
-    min_bounding_box = np.full(shape=(3, ), fill_value=np.inf)
-    max_bounding_box = np.full(shape=(3, ), fill_value=-np.inf)
+    """Get bounding box of a population"""
+    min_bounding_box = np.full(shape=(3,), fill_value=np.inf)
+    max_bounding_box = np.full(shape=(3,), fill_value=-np.inf)
 
     for population in populations:
         for morph in population:
             bounding_box = geom.bounding_box(morph)
             min_bounding_box = np.minimum(min_bounding_box, bounding_box[0][COLS.XYZ])
             max_bounding_box = np.maximum(max_bounding_box, bounding_box[1][COLS.XYZ])
 
-    return ((min_bounding_box[COLS.X], max_bounding_box[COLS.X]),
-            (min_bounding_box[COLS.Y], max_bounding_box[COLS.Y]),
-            )
+    return (
+        (min_bounding_box[COLS.X], max_bounding_box[COLS.X]),
+        (min_bounding_box[COLS.Y], max_bounding_box[COLS.Y]),
+    )
 
 
 def plot_population(output_dir, population, xlim, ylim, notebook_desc=None):
+    """Plot a population"""
     if not os.path.exists(output_dir):
         os.makedirs(output_dir)
 
     ret = []
     with pyplot_non_interactive():
-        for morph in add_progress_bar(population, '-- {}', notebook_desc):
+        for morph in add_progress_bar(population, "-- {}", notebook_desc):
             try:
                 fig, ax = viewer.draw(morph, diameter_scale=None)
                 ax.set_xlim(xlim)
                 ax.set_ylim(ylim)
-                file_name = os.path.join(output_dir, morph.name + '.png')
+                file_name = os.path.join(output_dir, morph.name + ".png")
                 fig.savefig(file_name)
                 plt.close(fig)
                 ret.append(file_name)
-            except:
-                print('Failed to plot neuron: ', morph.name)
+            except Exception:  # pylint: disable=broad-except
+                print("Failed to plot neuron: ", morph.name)
     return ret
 
 
-def plot_normalized_neurons(output_dir, ref_population, test_population,
-                            cell_figure_count, components, notebook_desc=None):
-    '''Plot cell examples and store the file in output_dir.
+def plot_normalized_neurons(
+    output_dir,
+    ref_population,
+    test_population,
+    cell_figure_count,
+    components,
+    notebook_desc=None,
+):
+    """Plot cell examples and store the file in output_dir.
 
     Args:
         output_dir(str): path to the output directory
@@ -170,32 +184,40 @@ def plot_normalized_neurons(output_dir, ref_population, test_population,
         cell_figure_count(int): number of exemplars to plot
         components(list of str): components to visualize
         notebook_desc(str): string used for labelling notebook progress bar
-    '''
+    """
     if not os.path.exists(output_dir):
         os.makedirs(output_dir)
 
     ref_population = _center_truncate_population(ref_population, cell_figure_count)
     test_population = _center_truncate_population(test_population, cell_figure_count)
 
-    ref_output_dir = os.path.join(output_dir, 'ref')
-    test_output_dir = os.path.join(output_dir, 'test')
+    ref_output_dir = os.path.join(output_dir, "ref")
+    test_output_dir = os.path.join(output_dir, "test")
 
     ref_plot_paths = {}
     test_plot_paths = {}
 
-    comp_dict = {'full_morph': 'morphologies'}
+    comp_dict = {"full_morph": "morphologies"}
     for comp in components:
-        comp_dict.update({comp: pretty_name(comp) + 's'})
+        comp_dict.update({comp: pretty_name(comp) + "s"})
 
-    for comp in add_progress_bar(comp_dict, '[{}] Plot morphologies', notebook_desc):
+    for comp in add_progress_bar(comp_dict, "[{}] Plot morphologies", notebook_desc):
         ref_comp_pop = get_components_population(ref_population, comp)
         test_comp_pop = get_components_population(test_population, comp)
         xlim, ylim = compute_bounding_box(ref_comp_pop, test_comp_pop)
 
-        desc = progress_bar_label('Reference {}', pretty_name(comp_dict[comp]), notebook_desc)
-        ref_plot_paths[comp] = plot_population(ref_output_dir, ref_comp_pop, xlim, ylim, desc)
-        desc = progress_bar_label('Test {}', pretty_name(comp_dict[comp]), notebook_desc)
-        test_plot_paths[comp] = plot_population(test_output_dir, test_comp_pop, xlim, ylim, desc)
+        desc = progress_bar_label(
+            "Reference {}", pretty_name(comp_dict[comp]), notebook_desc
+        )
+        ref_plot_paths[comp] = plot_population(
+            ref_output_dir, ref_comp_pop, xlim, ylim, desc
+        )
+        desc = progress_bar_label(
+            "Test {}", pretty_name(comp_dict[comp]), notebook_desc
+        )
+        test_plot_paths[comp] = plot_population(
+            test_output_dir, test_comp_pop, xlim, ylim, desc
+        )
 
     return ref_plot_paths, test_plot_paths
 

morphval/config.py

@@ -1,9 +1,11 @@
+"""config tools for MorphVal package"""
+
 import os
 
 import yaml
 
 
-EXAMPLE_CONFIG = '''\
+EXAMPLE_CONFIG = """\
 default: &default
   stat_test: 'StatTests.ks'
   threshold: 0.1
@@ -32,11 +34,12 @@
 config:
     L1_HAC:
         basal_dendrite: *basal_data
-'''
+"""
 
 
 def load_config(config_path):
-    assert os.path.exists(config_path), 'Missing config at: %s' % str(config_path)
+    """Load configuration from a YAML file"""
+    assert os.path.exists(config_path), "Missing config at: %s" % str(config_path)
     #  TODO: Should perform validation of config
-    with open(config_path, 'r') as fd:
-        return yaml.safe_load(fd)['config']
+    with open(config_path, "r") as fd:
+        return yaml.safe_load(fd)["config"]