feat: Better serialization mechanism allow for declaration data structure migration locally

.gitignore

@@ -479,7 +479,6 @@ test_data/
 /PartSegCore/segmentation/multiscale_opening/*.gch
 package/PartSegCore/multiscale_opening/\.vscode/
 
-package/PartSegCore/multiscale_opening/\.vscode/
 /tutorials/tutorial-chromosome-1/tutorial-chromosome1.pdf
 !/package/tests/test_data/measurements_profile.json
 
@@ -489,3 +488,6 @@ package/PartSegCore/multiscale_opening/\.vscode/
 *.e6t
 
 !docs/Makefile
+
+!package/tests/test_data/old_saves/*.[0-9]
+!package/tests/test_data/old_saves/*.[0-9][0-9]

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ default_language_version:
 
 repos:
 -   repo: https://github.com/psf/black
-    rev: 22.1.0
+    rev: 22.3.0
     hooks:
     - id: black
       pass_filenames: true

MANIFEST.in

@@ -2,6 +2,8 @@ global-include *.pyx
 global-include *.pxd
 include package/tests/test_data/ome.xsd.xml
 include package/tests/test_data/napari_measurements_profile.json
+include package/tests/test_data/notebook/*.json
+include package/tests/test_data/old_saves/*/*/*.json
 include Readme.md
 include changelog.md
 include pyproject.toml

docs/PartSegCore/base.rst

@@ -8,12 +8,6 @@ On this page there are described base classes of PartSegCore module
 .. automodule:: PartSegCore.algorithm_describe_base
    :members:
 
-.channel_class
---------------
-.. automodule:: PartSegCore.channel_class
-   :members:
-   :show-inheritance:
-
 .image_operations
 -----------------
 .. automodule:: PartSegCore.image_operations

docs/PartSegCore/class_register.rst

@@ -0,0 +1,9 @@
+PartSegCore.class_register
+==========================
+
+
+
+.. automodule:: PartSegCore.class_register
+   :members:
+   :inherited-members:
+   :show-inheritance:

docs/PartSegCore/index.rst

@@ -8,6 +8,7 @@ This package contains calculation backend of PartSeg
    :caption: Contents:
 
    base
+   class_register
    color_image
    segmentation
    analysis/base

docs/PartSegImage.rst

@@ -14,10 +14,3 @@ PartSegImage.image_reader
    :members:
    :exclude-members: ImageReader, MyTiffPage
    :show-inheritance:
-
-
-PartSegImage.tifffile_fixes
----------------------------
-
-.. autoclass:: PartSegImage.tifffile_fixes.MyTiffPage
-    :members: asarray

docs/conf.py

@@ -13,12 +13,15 @@
 # import os
 # import sys
 # sys.path.insert(0, os.path.abspath('.'))
+
+from datetime import date
+
 import PartSeg
 
 # -- Project information -----------------------------------------------------
 
 project = "PartSeg"
-copyright = "2019, Laboratory of Functional and Structural Genomics"
+copyright = f"{date.today().year}, Laboratory of Functional and Structural Genomics"
 author = "Grzegorz Bokota (LFSG)"
 
 # The full version, including alpha/beta/rc tags
@@ -37,7 +40,9 @@
     "sphinx_qt_documentation",
     "sphinx.ext.viewcode",
     "sphinx.ext.graphviz",
+    "sphinx_autodoc_typehints",
     "PartSegCore.sphinx.auto_parameters",
+    "sphinxcontrib.autodoc_pydantic",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -67,6 +72,7 @@
     "python": ("https://docs.python.org/3", None),
     "PyQt5": ("https://www.riverbankcomputing.com/static/Docs/PyQt5", None),
     "Numpy": ("https://docs.scipy.org/doc/numpy/", None),
+    "packaging": ("https://packaging.pypa.io/en/latest/", None),
 }
 
 qt_documentation = "Qt5"

docs/index.rst

@@ -15,7 +15,6 @@ The documentation is incomplete. Many utilities are undocumented.
    PartSegCore/index
    common_gui/index
    common_backend/index
-   custom_application
    PartSegImage
    interface-overview/interface-overview
 

docs/requirements.txt

@@ -1,3 +1,4 @@
-sphinx-autodoc-annotation==1.0.post1
+sphinx-autodoc-typehints==1.17.0
+autodoc_pydantic==1.6.1
 sphinx-qt-documentation==0.3
 sphinx!=3.0.0, !=3.5.0

package/PartSeg/_roi_analysis/advanced_window.py

@@ -2,7 +2,7 @@
 import os
 from contextlib import suppress
 from copy import deepcopy
-from typing import Optional, Tuple, Union
+from typing import Optional, Tuple, Union, cast
 
 from qtpy.QtCore import QEvent, Qt, Slot
 from qtpy.QtGui import QIcon
@@ -31,7 +31,7 @@
 from superqt import QEnumComboBox, ensure_main_thread
 
 from PartSeg.common_gui.advanced_tabs import AdvancedWindow
-from PartSegCore.analysis.algorithm_description import analysis_algorithm_dict
+from PartSegCore.analysis.algorithm_description import AnalysisAlgorithmSelection
 from PartSegCore.analysis.measurement_base import AreaType, Leaf, MeasurementEntry, Node, PerComponent
 from PartSegCore.analysis.measurement_calculation import MEASUREMENT_DICT, MeasurementProfile
 from PartSegCore.universal_const import UNIT_SCALE, Units
@@ -108,7 +108,7 @@ def __init__(self, settings: PartSettings):
         spacing_layout = QHBoxLayout()
         spacing_layout.addWidget(self.lock_spacing)
         for txt, el in zip(["x", "y", "z"], self.spacing[::-1]):
-            spacing_layout.addWidget(QLabel(txt + ":"))
+            spacing_layout.addWidget(QLabel(f"{txt}:"))
             spacing_layout.addWidget(el)
         spacing_layout.addWidget(self.units)
         spacing_layout.addStretch(1)
@@ -167,14 +167,15 @@ def profile_chosen(self, text):
         # TODO update with knowledge from profile dict
         self.delete_btn.setEnabled(True)
         self.rename_btn.setEnabled(True)
-        self.info_label.setPlainText(profile.pretty_print(analysis_algorithm_dict))
+        self.info_label.setPlainText(profile.pretty_print(AnalysisAlgorithmSelection.__register__))
 
     def synchronize_spacing(self):
         if self.lock_spacing.isChecked():
             self.spacing[1].setValue(self.spacing[2].value())
 
     def image_spacing_change(self):
-        spacing = [el.value() / UNIT_SCALE[self.units.currentIndex()] for i, el in enumerate(self.spacing)]
+        spacing = [el.value() / UNIT_SCALE[self.units.currentIndex()] for el in self.spacing]
+
         if not self.spacing[0].isEnabled():
             spacing = spacing[1:]
         self._settings.image_spacing = spacing
@@ -637,7 +638,7 @@ def get_parameters(self, node: Union[Node, Leaf], area: AreaType, component: Per
             node = node.replace_(per_component=component)
         with suppress(KeyError):
             arguments = MEASUREMENT_DICT[str(node.name)].get_fields()
-            if len(arguments) > 0 and len(node.dict) == 0:
+            if len(arguments) > 0 and len(dict(node.parameters)) == 0:
                 dial = self.form_dialog(arguments)
                 if dial.exec_():
                     node = node._replace(dict=dial.get_values())
@@ -697,9 +698,9 @@ def save_action(self):
                 return
         selected_values = []
         for i in range(self.profile_options_chosen.count()):
-            element: MeasurementListWidgetItem = self.profile_options_chosen.item(i)
-            selected_values.append(MeasurementEntry(element.text(), element.stat))
-        stat_prof = MeasurementProfile(self.profile_name.text(), selected_values)
+            element = cast(MeasurementListWidgetItem, self.profile_options_chosen.item(i))
+            selected_values.append(MeasurementEntry(name=element.text(), calculation_tree=element.stat))
+        stat_prof = MeasurementProfile(name=self.profile_name.text(), chosen_fields=selected_values)
         self.settings.measurement_profiles[stat_prof.name] = stat_prof
         self.settings.dump()
         self.export_profiles_butt.setEnabled(True)
@@ -722,9 +723,11 @@ def named_save_action(self):
         if val_dialog.exec_():
             selected_values = []
             for i in range(self.profile_options_chosen.count()):
-                element: MeasurementListWidgetItem = self.profile_options_chosen.item(i)
-                selected_values.append(MeasurementEntry(val_dialog.result[element.text()], element.stat))
-            stat_prof = MeasurementProfile(self.profile_name.text(), selected_values)
+                element = cast(MeasurementListWidgetItem, self.profile_options_chosen.item(i))
+                selected_values.append(
+                    MeasurementEntry(name=val_dialog.result[element.text()], calculation_tree=element.stat)
+                )
+            stat_prof = MeasurementProfile(name=self.profile_name.text(), chosen_fields=selected_values)
             self.settings.measurement_profiles[stat_prof.name] = stat_prof
             self.export_profiles_butt.setEnabled(True)
 

package/PartSeg/_roi_analysis/main_window.py

@@ -104,7 +104,7 @@ def __init__(
         self.choose_profile.textActivated.connect(self.change_profile)
         self.interactive_use.stateChanged.connect(self.execute_btn.setDisabled)
         self.interactive_use.stateChanged.connect(self.interactive_change)
-        self.algorithm_choose_widget = AlgorithmChoose(settings, algorithm_description.analysis_algorithm_dict)
+        self.algorithm_choose_widget = AlgorithmChoose(settings, algorithm_description.AnalysisAlgorithmSelection)
         self.algorithm_choose_widget.result.connect(self.execution_done)
         self.algorithm_choose_widget.finished.connect(self.calculation_finished)
         self.algorithm_choose_widget.value_changed.connect(self.interactive_algorithm_execute)
@@ -627,7 +627,7 @@ def image_read(self):
         self.options_panel.interactive_algorithm_execute()
 
     def reload(self):
-        self.options_panel.algorithm_choose_widget.reload(algorithm_description.analysis_algorithm_dict)
+        self.options_panel.algorithm_choose_widget.reload(algorithm_description.AnalysisAlgorithmSelection)
 
     def closeEvent(self, event):
         self.settings.set_in_profile("main_window_geometry", self.saveGeometry().toHex().data().decode("ascii"))

package/PartSeg/_roi_analysis/partseg_settings.py

@@ -10,11 +10,11 @@
 from PartSegCore.analysis.io_utils import MaskInfo, ProjectTuple
 from PartSegCore.analysis.load_functions import load_metadata
 from PartSegCore.analysis.measurement_calculation import MeasurementProfile
-from PartSegCore.analysis.save_hooks import PartEncoder
 from PartSegCore.io_utils import PointsInfo
-from PartSegCore.json_hooks import EventedDict, ProfileDict
+from PartSegCore.json_hooks import PartSegEncoder
 from PartSegCore.project_info import HistoryElement
 from PartSegCore.roi_info import ROIInfo
+from PartSegCore.utils import EventedDict, ProfileDict
 
 from ..common_backend.base_settings import BaseSettings, SaveSettingsDescription
 
@@ -29,7 +29,7 @@ class PartSettings(BaseSettings):
     roi_pipelines_changed = Signal()
     measurement_profiles_changed = Signal()
     batch_plans_changed = Signal()
-    json_encoder_class = PartEncoder
+    json_encoder_class = PartSegEncoder
     load_metadata = staticmethod(load_metadata)
     last_executed_algorithm: str
     save_locations_keys = [
@@ -60,8 +60,8 @@ def fix_history(self, algorithm_name, algorithm_values):
         :param str algorithm_name:
         :param dict algorithm_values:
         """
-        self.history[self.history_index + 1] = self.history[self.history_index + 1].replace_(
-            roi_extraction_parameters={"algorithm_name": algorithm_name, "values": algorithm_values}
+        self.history[self.history_index + 1] = self.history[self.history_index + 1].copy(
+            update=dict(roi_extraction_parameters={"algorithm_name": algorithm_name, "values": algorithm_values})
         )
 
     @staticmethod

package/PartSeg/_roi_analysis/prepare_plan_widget.py

@@ -34,7 +34,7 @@
 from superqt import QEnumComboBox
 
 from PartSegCore.algorithm_describe_base import AlgorithmProperty, ROIExtractionProfile
-from PartSegCore.analysis.algorithm_description import analysis_algorithm_dict
+from PartSegCore.analysis.algorithm_description import AnalysisAlgorithmSelection
 from PartSegCore.analysis.calculation_plan import (
     CalculationPlan,
     MaskBase,
@@ -599,7 +599,7 @@ def create_mask(self):
                 return
             names = dial.get_result()
 
-            mask_ob = MaskConstruct(text, *names)
+            mask_ob = MaskConstruct(name=text, mask1=names[0], mask2=names[1])
         else:
             raise ValueError("Unknowsn widget")
 
@@ -868,7 +868,7 @@ def show_segment_info(self, text=None):
                 else:
                     return
             profile = self.settings.roi_pipelines[text]
-        self.information.setText(profile.pretty_print(analysis_algorithm_dict))
+        self.information.setText(profile.pretty_print(AnalysisAlgorithmSelection))
 
     def show_segment(self):
         if self.update_element_chk.isChecked() and self.segment_stack.currentIndex() == 0:
@@ -963,7 +963,7 @@ def explore_tree(self, up_widget, node_plan, deep=True):
             desc = QTreeWidgetItem(widget)
             desc.setText(0, "Description")
             if isinstance(node_plan.operation, ROIExtractionProfile):
-                txt = node_plan.operation.pretty_print(analysis_algorithm_dict)
+                txt = node_plan.operation.pretty_print(AnalysisAlgorithmSelection)
             else:
                 txt = str(node_plan.operation)
             for line in txt.split("\n")[1:]:
@@ -1015,7 +1015,7 @@ def update_view(self, reset=False):
                     child = node.child(0)
                     child.takeChildren()
                     if isinstance(el.operation, ROIExtractionProfile):
-                        txt = el.operation.pretty_print(analysis_algorithm_dict)
+                        txt = el.operation.pretty_print(AnalysisAlgorithmSelection)
                     else:
                         txt = str(el.operation)
                     for line in txt.split("\n")[1:]:

package/PartSeg/_roi_analysis/profile_export.py

@@ -19,7 +19,7 @@
 
 from PartSeg.common_gui.searchable_list_widget import SearchableListWidget
 from PartSegCore.algorithm_describe_base import ROIExtractionProfile
-from PartSegCore.analysis.algorithm_description import analysis_algorithm_dict
+from PartSegCore.analysis.algorithm_description import AnalysisAlgorithmSelection
 
 
 class ExportDialog(QDialog):
@@ -314,5 +314,5 @@ class ProfileDictViewer(ObjectPreview):
     """
 
     def preview_object(self, ob: ROIExtractionProfile):
-        text = ob.pretty_print(analysis_algorithm_dict)
+        text = ob.pretty_print(AnalysisAlgorithmSelection.__register__)
         self.setText(text)

package/PartSeg/_roi_mask/batch_proceed.py

@@ -9,7 +9,7 @@
 
 from PartSeg._roi_mask.stack_settings import StackSettings, get_mask
 from PartSegCore.algorithm_describe_base import ROIExtractionProfile
-from PartSegCore.mask.algorithm_description import mask_algorithm_dict
+from PartSegCore.mask.algorithm_description import MaskAlgorithmSelection
 from PartSegCore.mask.io_functions import LoadROIImage, LoadStackImage, MaskProjectTuple, SaveROI
 from PartSegCore.segmentation import StackAlgorithm
 from PartSegCore.segmentation.algorithm_base import ROIExtractionAlgorithm
@@ -66,7 +66,7 @@ def run_calculation(self):
             try:
                 name = path.basename(file_path)
                 blank = get_mask(project_tuple.roi, project_tuple.mask, project_tuple.selected_components)
-                algorithm: StackAlgorithm = mask_algorithm_dict[task.parameters.algorithm]()
+                algorithm: StackAlgorithm = MaskAlgorithmSelection[task.parameters.algorithm]()
                 algorithm.set_image(project_tuple.image)
                 algorithm.set_mask(blank)
                 algorithm.set_parameters(**task.parameters.values)
@@ -81,7 +81,7 @@ def run_calculation(self):
                 )
                 if isinstance(task.save_prefix, tuple):
                     self.progress_info(name, "saving", algorithm.get_steps_num())
-                    name = path.splitext(path.basename(file_path))[0] + ".seg"
+                    name = f"{path.splitext(path.basename(file_path))[0]}.seg"
                     re_end = re.compile(r"(.*_version)(\d+)\.seg$")
                     while path.exists(path.join(task.save_prefix[0], name)):
                         match = re_end.match(name)