Use pytest template to improve tests and coverage reports

.coveragerc

@@ -0,0 +1,8 @@
+[paths]
+source_synthesis =
+	src/synthesis_workflow
+	*/site-packages/synthesis_workflow
+
+source_morphval =
+	src/morphval
+	*/site-packages/morphval

.gitlab-ci.yml

@@ -5,8 +5,6 @@ include:
         - /ci/jobs/check-version.yml
         - /ci/jobs/check-packaging.yml
         - /ci/jobs/lint.yml
-        - /ci/jobs/py37.yml
-        - /ci/jobs/py38.yml
         - /ci/jobs/docs.yml
         - /ci/jobs/build-package.yml
         - /ci/jobs/publish-package.yml
@@ -15,6 +13,10 @@ include:
    - project: neuromath/ci
      file:
         - /ci/jobs/auto-release.yml
+        - /ci/lib/pytest-template.yml
+        - /ci/jobs/py37.yml
+        - /ci/jobs/py38.yml
+        - /ci/jobs/coverage.yml
 
 .tox-template:
   # Using the same image as DVF

doc/source/conf.py

@@ -14,17 +14,16 @@
 
 import importlib
 import re
-from pkg_resources import get_distribution
 
 import luigi
+from pkg_resources import get_distribution
 
 import morphval
 import synthesis_workflow
 import synthesis_workflow.tasks
 from synthesis_workflow.tasks.cli import _PARAM_NO_VALUE
 from synthesis_workflow.tasks.cli import _process_param
 
-
 # -- Project information -----------------------------------------------------
 
 project = "synthesis-workflow"

.pylintrc → pyproject.toml

@@ -1,30 +1,34 @@
-[MASTER]
-ignore=version.py
+# BLACK
+[tool.black]
+line-length = 100
+target-version = ["py36"]
 
-[MESSAGES CONTROL]
-disable=
-	bad-continuation,
-	duplicate-code,
-	empty-docstring,
-	fixme,
-	invalid-name,
-	len-as-condition,
-	no-else-return,
-	no-member,
-	too-many-ancestors,
-	too-many-lines,
-    unsubscriptable-object,
+# PYLINT
+[tool.pylint.messages-control]
+disable=[
+	"bad-continuation",
+	"duplicate-code",
+	"empty-docstring",
+	"fixme",
+	"invalid-name",
+	"len-as-condition",
+	"no-else-return",
+	"no-member",
+	"too-many-ancestors",
+	"too-many-lines",
+    "unsubscriptable-object",
+]
 
-[FORMAT]
+[tool.pylint.format]
 # Maximum number of characters on a single line.
 max-line-length=100
 
-[DESIGN]
+[tool.pylint.design]
 # Maximum number of arguments for function / method
 max-args=10
 # Argument names that match this expression will be ignored. Default to name
 # with leading underscore
-ignored-argument-names=args|kwargs|_.*
+ignored-argument-names="args|kwargs|_.*"
 # Maximum number of locals for function / method body
 max-locals=25
 # Maximum number of return / yield for function / method body
@@ -34,37 +38,47 @@ max-branches=20
 # Maximum number of statements in function / method body
 max-statements=60
 # Maximum number of parents for a class (see R0901).
-max-parents=7
+max-parents=10
 # Maximum number of attributes for a class (see R0902).
 max-attributes=40
 # Minimum number of public methods for a class (see R0903).
 min-public-methods=0
 # Maximum number of public methods for a class (see R0904).
 max-public-methods=60
-# checks for similarities and duplicated code. This computation may be
-# memory / CPU intensive, so you should disable it if you experiments some
-# problems.
-#
 
-[SIMILARITIES]
+[tool.pylint.similarities]
 # Minimum lines number of a similarity.
 min-similarity-lines=25
 # Ignore comments when computing similarities.
-ignore-comments=yes
+ignore-comments=true
 # Ignore docstrings when computing similarities.
-ignore-docstrings=yes
+ignore-docstrings=true
 
-[TYPECHECK]
-# List of classes names for which member attributes should not be checked
-# (useful for classes with attributes dynamically set).
-
-#as of numpy 1.8.0, name resolution seems to be a problem.  Ignore lookups in numpy
-ignored-classes=numpy,list
-
-extension-pkg-whitelist=numpy,lxml
+[tool.pylint.typecheck]
+extension-pkg-whitelist = ["numpy", "lxml"]
 
 # List of module names for which member attributes should not be checked
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis. It
 # supports qualified module names, as well as Unix pattern matching.
-ignored-modules=atlas_analysis,atlas_analysis.constants
+ignored-modules=["atlas_analysis", "atlas_analysis.constants"]
+
+# PYDOCSTYLE
+[tool.pydocstyle]
+# ignore the following:
+#   - D102: Missing docstring in public method
+#   - D105: Missing docstring in magic method
+#   - D107: Missing docstring in __init__
+add-ignore = [
+	"D102",
+	"D105",
+	"D107",
+]
+convention = "google"
+
+# ISORT
+[tool.isort]
+line_length = 100
+profile = "black"
+force_single_line = true
+

setup.py

@@ -1,9 +1,9 @@
-"""Setup"""
-#!/usr/bin/env python3
+"""Setup for the synthesis-workflow package."""
 import importlib
 import sys
 
-from setuptools import setup, find_packages
+from setuptools import find_packages
+from setuptools import setup
 
 if sys.version_info < (3, 6):
     sys.exit("Sorry, Python < 3.6 is not supported")

src/morphval/cli.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python
 """CLI for MorphVal package."""
 import argparse
+
 import pkg_resources
 
-from morphval.validation_main import Validation
 from morphval import config
+from morphval.validation_main import Validation
 
 
 def get_parser():

src/morphval/common.py

@@ -1,23 +1,22 @@
 """Private helper functions of the validation module."""
 import contextlib
-import os
 import json
+import os
 from copy import deepcopy
 
-from neurom import viewer, geom, NeuriteType
+import numpy as np
+from matplotlib import pyplot as plt
+from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
+from matplotlib.figure import Figure
+from neurom import NeuriteType
+from neurom import geom
+from neurom import viewer
 from neurom.core.dataformat import COLS
 from neurom.core.neuron import iter_neurites
 from neurom.core.population import Population
 from neurom.core.types import tree_type_checker as is_type
 from region_grower.utils import NumpyEncoder
 
-import numpy as np
-
-from matplotlib import pyplot as plt
-from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
-from matplotlib.figure import Figure
-
-
 COMP_MAP = {
     "basal_dendrite": NeuriteType.basal_dendrite,
     "apical_dendrite": NeuriteType.apical_dendrite,

src/morphval/validation.py

@@ -4,9 +4,8 @@
 from decimal import Decimal
 
 import numpy as np
-
-from neurom import stats
 from neurom import get
+from neurom import stats
 from neurom.core.types import NeuriteType
 
 DICTDATA = dict.fromkeys(["name", "data_type", "data", "labels"])

src/morphval/validation_main.py

@@ -9,14 +9,13 @@
 import numpy as np
 import pandas as pd
 import pkg_resources
-from joblib import delayed
 from joblib import Parallel
 from joblib import cpu_count
+from joblib import delayed
 
 from morphval import common
 from morphval import validation
 
-
 TEMPLATE_FILE = pkg_resources.resource_filename("morphval", "templates/report_template.jinja2")
 SUMMARY_TEMPLATE_FILE = pkg_resources.resource_filename(
     "morphval", "templates/report_summary_template.jinja2"

src/synthesis_workflow/__init__.py

@@ -2,7 +2,6 @@
 import pkg_resources
 from morphio import SectionType
 
-
 __version__ = pkg_resources.get_distribution("synthesis_workflow").version
 
 STR_TO_TYPES = {

src/synthesis_workflow/circuit.py

@@ -1,13 +1,13 @@
 """Functions for slicing mvd3 circuit files to place specific cells only."""
 import logging
+
+import numpy as np
 import pandas as pd
+from atlas_analysis.planes.planes import create_planes as _create_planes
+from brainbuilder.app.cells import _place as place
 from tqdm import tqdm
 from voxcell import CellCollection
 from voxcell.nexus.voxelbrain import LocalAtlas
-import numpy as np
-
-from atlas_analysis.planes.planes import create_planes as _create_planes
-from brainbuilder.app.cells import _place as place
 
 L = logging.getLogger(__name__)
 LEFT = 0

src/synthesis_workflow/extra/insitu_validation.py

@@ -1,20 +1,21 @@
 """In-situ validation functions for synthesis in atlas."""
 from pathlib import Path
-from tqdm import tqdm
-import numpy as np
-import pandas as pd
 
-import matplotlib.pyplot as plt
 import matplotlib
-from matplotlib.backends.backend_pdf import PdfPages
-import seaborn as sns
-
-from morphio import Morphology, SectionType
+import matplotlib.pyplot as plt
 import neurom as nm
+import numpy as np
+import pandas as pd
+import seaborn as sns
+from bluepyparallel import evaluate
+from bluepyparallel import init_parallel_factory
+from matplotlib.backends.backend_pdf import PdfPages
+from morphio import Morphology
+from morphio import SectionType
+from region_grower.atlas_helper import AtlasHelper
+from tqdm import tqdm
 from voxcell import CellCollection
 from voxcell.nexus.voxelbrain import Atlas
-from region_grower.atlas_helper import AtlasHelper
-from bluepyparallel import evaluate, init_parallel_factory
 
 from synthesis_workflow.tools import get_layer_tags
 

src/synthesis_workflow/extra/make_map.py

@@ -1,12 +1,11 @@
 """Make a atlas map from scatter data."""
 from copy import copy
-from pyquaternion import Quaternion
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
 
 import matplotlib
-
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+from pyquaternion import Quaternion
 
 from synthesis_workflow.validation import get_annotation_info
 

src/synthesis_workflow/fit_utils.py

@@ -1,9 +1,10 @@
 """Some functions used to fit path distances with depth."""
-from typing import Tuple, Sequence
+from typing import Sequence
+from typing import Tuple
 
 import numpy as np
-from scipy.optimize import curve_fit
 import tmd
+from scipy.optimize import curve_fit
 from tmd.Population.Population import Population
 
 

src/synthesis_workflow/morphology_combos/create_me_combos.py

@@ -1,9 +1,9 @@
 """Functions to create and mofify combos dataframe."""
-from pathlib import Path
 import json
 import logging
-import pandas as pd
+from pathlib import Path
 
+import pandas as pd
 from bluepymm.prepare_combos.parse_files import read_mm_recipe
 
 L = logging.getLogger(__name__)

src/synthesis_workflow/synthesis.py

@@ -8,25 +8,23 @@
 import matplotlib
 import numpy as np
 import pandas as pd
-from joblib import delayed
 from joblib import Parallel
-from tqdm import tqdm
-
+from joblib import delayed
 from morphio.mut import Morphology
-from neuroc.scale import scale_section
 from neuroc.scale import ScaleParameters
+from neuroc.scale import scale_section
 from neurom.core.dataformat import COLS
 from placement_algorithm.app import utils
 from placement_algorithm.app.choose_morphologies import Master as ChooseMorphologyMaster
 from tmd.io.io import load_population
 from tns import extract_input
+from tqdm import tqdm
 from voxcell import CellCollection
 
 from synthesis_workflow import STR_TO_TYPES
 from synthesis_workflow.fit_utils import fit_path_distance_to_extent
 from synthesis_workflow.tools import run_master
 
-
 L = logging.getLogger(__name__)
 matplotlib.use("Agg")