Mass plotting notebook (#1052)

* added plotting notebook

* updated readmed

* git merge issues

* git merge fixes v2 - docs/landing.md

* type

* Remove cells that don't have any pixel clusters expressed prior to Pixie (#1051)

* Drop cells without any SOM cluster expressions

* Add testing for dropping non-expressed cells

* Include `img_sub_folder` as parameter in Mantis calls (#1050)

* Ensure mantis can read the image sub folder specified

* Test on example dataset

* Default img_sub_folder to None in create_mantis_dir (standard used elsewhere)

* Doc fix and updated example dataset

* Update segmentation labels again

* Clarify img_sub_folder (mostly for refreshing tests)

* Force .github/get_example_dataset.py to use updated revision

* Update the file names to look for

* fixed clipped plots in the notebooks.

* Update HuggingFace version (#1053)

* v0.1.10 (#1057)

* Added continuous variable segmentation plots

* Pin to scikit-image to v0.19.3 (#1060)

* pin to 0.19.3

* requirement less than 0.19.3

* Make sure overwrite functionality for pixel clustering propagates into helper functions (#1058)

* Propagate overwrite functionality for pixel clustering

* Ensure re-normalization prevented on overwrite for pixel SOM re-assignment

* normalize_data should be the opposite of overwrite

* Single mantis directory (#1061)

* single mantis dir, no recopying

* post clustering mask suffix

* generic clustering adjustments

* mask prefix

* remove mask prefix

* strange merge issue

* removed branch filters on CI, so if you branch off of a branch and run GHA, the CI workflow should work

* changed ci.yml, filter push on main branch only

* python 3.9 doesn't support | for Union type overloading

* added fovs parameter for continuous variable plots

* updated notebook to include plots

* updated cell clustering notebooks to add erosion (default to True), cleaned up some notebook outputs

* fixed style imports

* re-added test_generate_summary_stats. Was accidently deleted in a refactoring step.

* newline at eof fiber_segmentation_test.py

* seaborn-paper -> seaborn-v0_8-paper

* added cell to view all plotting styles i mask gen nb

* pycodestyle

---------

Co-authored-by: alex-l-kong <31424707+alex-l-kong@users.noreply.github.com>
Co-authored-by: camisowers <38049893+camisowers@users.noreply.github.com>

.github/workflows/ci.yml

@@ -4,12 +4,10 @@ on:
   push:
     branches: [main]
   pull_request:
-    branches: [main]
     types: [labeled, opened, synchronize, reopened]
   workflow_dispatch:
   merge_group:
     types: [checks_requested]
-    branches: [main]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}

.gitignore

@@ -40,3 +40,4 @@ data/*
 .venv
 .venv/
 .virtual_documents
+.history/

environment.yml

@@ -6,4 +6,4 @@ dependencies:
   - pip
   - hdf5
   - pip:
-    - .[lab_ext]
+    - .[lab_ext, colors]

pyproject.toml

@@ -39,6 +39,7 @@ dependencies = [
     "xarray>=2022",
     "xmltodict>=0.13.0,<1",
     "zstandard>=0.19.0,<1",
+    "ark-analysis[colors]",
 ]
 name = "ark-analysis"
 authors = [{ name = "Angelo Lab", email = "theangelolab@gmail.com" }]
@@ -59,6 +60,13 @@ dynamic = ["version"]
 urls = { repository = "https://github.com/angelolab/ark-analysis" }
 
 [project.optional-dependencies]
+colors = [
+    "palettable",
+    "cmasher",
+    "cmocean",
+    "colorcet",
+    "scienceplots"
+]
 test = [
     "attrs",
     "coveralls[toml]",

src/ark/phenotyping/post_cluster_utils.py

@@ -122,7 +122,7 @@ def create_mantis_project(
     # generate unique numeric value for each population
     small_table["pop_vals"] = pd.factorize(small_table[pop_col].tolist())[0] + 1
 
-    ccmd_pop = data_utils.CellClusterMaskData(
+    cmd_pop = data_utils.ClusterMaskData(
         data=small_table,
         fov_col=fov_col,
         label_col=label_col,
@@ -145,8 +145,8 @@ def create_mantis_project(
         # use label_cells_by_cluster to create cell masks
         mask_data = data_utils.label_cells_by_cluster(
             fov=fov,
-            ccmd=ccmd_pop,
-            label_map=label_map,
+            cmd=cmd_pop,
+            label_map=label_map.values,
         )
         # save the cell mask for each FOV -- (saves with ".tiff" extension)
         data_utils.save_fov_mask(

src/ark/phenotyping/weighted_channel_comp.py

@@ -42,6 +42,9 @@ def compute_p2c_weighted_channel_avg(pixel_channel_avg, channels, cell_counts,
         pandas.DataFrame:
             Returns the average marker expression for each cell in the dataset
     """
+
+    if "segmentation_label" in cell_counts.columns:
+        cell_counts.rename(columns={"segmentation_label": "label"}, inplace=True)
 
     # if no fovs provided make sure they're all iterated over
     if fovs is None:
@@ -123,8 +126,6 @@ def compute_p2c_weighted_channel_avg(pixel_channel_avg, channels, cell_counts,
     weighted_cell_channel = pd.DataFrame(
         weighted_cell_channel, columns=channels
     )
-    if "segmentation_label" in weighted_cell_channel.columns:
-        weighted_cell_channel.rename({"segmentation_label": "label"}, inplace=True)
 
     # add columns back
     meta_cols = ['cell_size', 'fov', 'label']

src/ark/segmentation/fiber_segmentation.py

@@ -1,6 +1,7 @@
 import itertools
 import os
-from typing import Dict, Optional
+import pathlib
+from typing import Dict, Optional, Union
 
 import matplotlib.pyplot as plt
 import natsort as ns
@@ -67,9 +68,9 @@ def plot_fiber_segmentation_steps(data_dir, fov_name, fiber_channel, img_sub_fol
                               all_channels=io_utils.remove_file_extensions(
                                   io_utils.list_files(
                                       os.path.join(data_dir, fov_name, img_sub_folder)
-                                      )
                                   )
-                              )
+    )
+    )
 
     data_xr = load_utils.load_imgs_from_tree(
         data_dir, img_sub_folder, fovs=[fov_name], channels=[fiber_channel]
@@ -171,9 +172,9 @@ def run_fiber_segmentation(data_dir, fiber_channel, out_dir, img_sub_folder=None
                               all_channels=io_utils.remove_file_extensions(
                                   io_utils.list_files(
                                       os.path.join(data_dir, fovs[0], img_sub_folder)
-                                      )
                                   )
-                              )
+    )
+    )
 
     fiber_object_table = []
 
@@ -246,7 +247,7 @@ def calculate_fiber_alignment(fiber_object_table, k=4, axis_thresh=2):
             # find index for smallest distances, excluding itself
             indy = fiber_dist_mat[indx, :].argsort()[1:1+k]
             neighbor_angles = filtered_lengths.orientation[indy]
-            fiber_scores.append(1 / (np.sqrt(np.sum((neighbor_angles - angle) ** 2)) / k))
+            fiber_scores.append((np.sqrt(np.sum((neighbor_angles - angle) ** 2)) / k))
 
         fov_alignments = pd.DataFrame(
             zip([fov] * len(fiber_scores), filtered_lengths.label, fiber_scores),
@@ -554,7 +555,7 @@ def generate_summary_stats(fiber_object_table, fibseg_dir, tile_length=512, min_
         'fov': fovs,
         'pixel_density': fov_pixel_density,
         'fiber_density': fov_fiber_density
-        })
+    })
 
     fov_prop_stats = np.vstack(fov_avg_stats)
     for i, metric in enumerate(properties):
@@ -567,51 +568,3 @@ def generate_summary_stats(fiber_object_table, fibseg_dir, tile_length=512, min_
                       index=False)
 
     return fov_stats, tile_stats
-
-
-def color_fibers_by_stat(fiber_object_table, fibseg_dir, save_dir, stat_name):
-    """ Creates colored fiber masks based on values from a user-specified column of the
-    fiber_object_table.
-
-    Args:
-        fiber_object_table (pd.DataFrame):
-            dataframe containing the fiber objects and their properties (fov, label, alignment,
-            centroid-0, centroid-1, major_axis_length, minor_axis_length)
-        fibseg_dir (string):
-            path to directory containing the fiber segmentation masks
-        save_dir (str):
-            where to save colored masks to
-        stat_name (int):
-            name of the column to use for fiber coloring
-       """
-
-    io_utils.validate_paths(fibseg_dir)
-    misc_utils.verify_in_list(statistic_name=[stat_name],
-                              fiber_table_columns=fiber_object_table.columns)
-    if not os.path.exists(save_dir):
-        os.makedirs(save_dir)
-
-    cmap = set_minimum_color_for_colormap(plt.cm.Blues)
-    # alignment score needs reversed colormap after taking the inverse
-    if stat_name == 'alignment_score':
-        cmap = set_minimum_color_for_colormap(plt.cm.Blues_r)
-
-    for fov in np.unique(fiber_object_table.fov):
-        fiber_data = fiber_object_table[fiber_object_table.fov == fov]
-        fov_fiber_img = io.imread(os.path.join(fibseg_dir, fov + '_fiber_labels.tiff'))
-        fov_fiber_img = fov_fiber_img.astype('float16')
-
-        # reassign the fiber mask values with the stat values
-        for fiber in fiber_data.label:
-            stat = fiber_data.loc[fiber_data.label == fiber][stat_name].values[0]
-
-            # ignore any fibers without stat value
-            if np.isnan(stat):
-                stat = 0
-            # use inverse of alignment score
-            elif stat_name == 'alignment_score':
-                stat = 1 / stat
-
-            fov_fiber_img[fov_fiber_img == fiber] = stat
-
-        plt.imsave(os.path.join(save_dir, fov + f"_{stat_name}.tiff"), fov_fiber_img, cmap=cmap)