Update progress bars

src/ark/analysis/cell_neighborhood_stats.py

@@ -49,8 +49,11 @@ def compute_neighborhood_diversity(neighborhood_mat, cell_type_col):
 
     diversity_data = []
     fov_list = np.unique(neighborhood_mat[settings.FOV_ID])
-    with tqdm(total=len(fov_list), desc="Calculate Neighborhood Diversity") as diversity_progress:
+    with tqdm(total=len(fov_list), desc="Calculate Neighborhood Diversity", unit="FOVs") \
+            as diversity_progress:
         for fov in fov_list:
+            diversity_progress.set_postfix(FOV=fov)
+
             fov_neighborhoods = neighborhood_mat[neighborhood_mat[settings.FOV_ID] == fov]
 
             diversity_scores = []
@@ -72,7 +75,6 @@ def compute_neighborhood_diversity(neighborhood_mat, cell_type_col):
             })
             diversity_data.append(fov_data)
 
-            diversity_progress.set_postfix(FOV=fov)
             diversity_progress.update(1)
 
     # dataframe containing all fovs
@@ -216,8 +218,11 @@ def generate_cell_distance_analysis(
     fov_list = np.unique(cell_table[fov_col])
 
     cell_dists = []
-    with tqdm(total=len(fov_list), desc="Calculate Average Distances") as distance_progress:
+    with tqdm(total=len(fov_list), desc="Calculate Average Distances", unit="FOVs") \
+            as distance_progress:
         for fov in fov_list:
+            distance_progress.set_postfix(FOV=fov)
+
             fov_cell_table = cell_table[cell_table[fov_col] == fov]
             fov_dist_xr = xr.load_dataarray(os.path.join(dist_mat_dir, str(fov) + '_dist_mat.xr'))
 
@@ -231,7 +236,6 @@ def generate_cell_distance_analysis(
             fov_cell_dists.insert(2, cell_type_col, fov_cell_table[cell_type_col])
             cell_dists.append(fov_cell_dists)
 
-            distance_progress.set_postfix(FOV=fov)
             distance_progress.update(1)
 
     # combine data for all fovs and save to csv

src/ark/analysis/neighborhood_analysis.py

@@ -76,9 +76,11 @@ def create_neighborhood_matrix(all_data, dist_mat_dir, included_fovs=None, distl
 
     cell_neighbor_freqs = cell_neighbor_counts.copy(deep=True)
 
-    with tqdm(total=len(included_fovs), desc="Neighbors Matrix Generation") \
+    with tqdm(total=len(included_fovs), desc="Neighbors Matrix Generation", unit="FOVs") \
             as neighbor_mat_progress:
         for fov in included_fovs:
+            neighbor_mat_progress.set_postfix(FOV=fov)
+
             # Subsetting expression matrix to only include patients with correct fov label
             current_fov_idx = all_neighborhood_data.loc[:, fov_col] == fov
             current_fov_neighborhood_data = all_neighborhood_data[current_fov_idx]
@@ -100,7 +102,6 @@ def create_neighborhood_matrix(all_data, dist_mat_dir, included_fovs=None, distl
             cell_neighbor_freqs.loc[current_fov_neighborhood_data.index, fov_cluster_names]\
                 = freqs
 
-            neighbor_mat_progress.set_postfix(FOV=fov)
             neighbor_mat_progress.update(1)
 
     # Remove cells that have no neighbors within the distlim

src/ark/analysis/spatial_analysis_utils.py

@@ -37,8 +37,11 @@ def calc_dist_matrix(label_dir, save_path, prefix='_whole_cell'):
     fov_files = io_utils.list_files(label_dir, substrs=prefix + '.tiff')
 
     # iterate for each fov
-    with tqdm(total=len(fov_files), desc="Distance Matrix Generation") as dist_mat_progress:
+    with tqdm(total=len(fov_files), desc="Distance Matrix Generation", unit="FOVs") \
+            as dist_mat_progress:
         for fov_file in fov_files:
+            dist_mat_progress.set_postfix(FOV=fov_file)
+
             # retrieve the fov name
             fov_name = fov_file.replace(prefix + '.tiff', '')
 
@@ -66,7 +69,6 @@ def calc_dist_matrix(label_dir, save_path, prefix='_whole_cell'):
                 format='NETCDF3_64BIT'
             )
 
-            dist_mat_progress.set_postfix(FOV=fov_name)
             dist_mat_progress.update(1)
 
 

src/ark/analysis/spatial_enrichment.py

@@ -70,8 +70,11 @@ def generate_channel_spatial_enrichment_stats(label_dir, dist_mat_dir, marker_th
     values = []
     stats_datasets = []
 
-    with tqdm(total=len(all_label_fovs), desc="Channel Spatial Enrichment") as chan_progress:
+    with tqdm(total=len(all_label_fovs), desc="Channel Spatial Enrichment", unit="FOVs") \
+            as chan_progress:
         for fov_name, label_file in zip(all_label_fovs, all_label_names):
+            chan_progress.set_postfix(FOV=fov_name)
+
             label_maps = load_utils.load_imgs_from_dir(label_dir, files=[label_file],
                                                        xr_channel_names=[xr_channel_name],
                                                        trim_suffix=suffix)
@@ -292,8 +295,11 @@ def generate_cluster_spatial_enrichment_stats(label_dir, dist_mat_dir, all_data,
     values = []
     stats_datasets = []
 
-    with tqdm(total=len(all_label_fovs), desc="Cluster Spatial Enrichment") as clust_progress:
+    with tqdm(total=len(all_label_fovs), desc="Cluster Spatial Enrichment", unit="FOVs") \
+            as clust_progress:
         for fov_name, label_file in zip(all_label_fovs, all_label_names):
+            clust_progress.set_postfix(FOV=fov_name)
+
             label_maps = load_utils.load_imgs_from_dir(label_dir, files=[label_file],
                                                        xr_channel_names=[xr_channel_name],
                                                        trim_suffix=suffix)

src/ark/segmentation/fiber_segmentation.py

@@ -16,6 +16,7 @@
 from skimage.measure import regionprops_table
 from skimage.morphology import remove_small_objects
 from skimage.segmentation import watershed
+from tqdm.auto import tqdm
 
 from ark import settings
 from ark.utils.plot_utils import set_minimum_color_for_colormap
@@ -176,14 +177,21 @@ def run_fiber_segmentation(data_dir, fiber_channel, out_dir, img_sub_folder=None
 
     fiber_object_table = []
 
-    for fov in fovs:
-        print(f'Processing FOV: {fov}')
-        subset_xr = load_utils.load_imgs_from_tree(
-            data_dir, img_sub_folder, fovs=fov, channels=[fiber_channel]
-        )
-        subtable = segment_fibers(subset_xr, fiber_channel, out_dir, fov, save_csv=False,
-                                  **kwargs)
-        fiber_object_table.append(subtable)
+    with tqdm(total=len(fovs), desc="Fiber Segmentation", unit="FOVs") \
+            as fibseg_progress:
+        for fov in fovs:
+            fibseg_progress.set_postfix(FOV=fov)
+
+            subset_xr = load_utils.load_imgs_from_tree(
+                data_dir, img_sub_folder, fovs=fov, channels=[fiber_channel]
+            )
+            # run fiber segmentation on the FOV
+            subtable = segment_fibers(subset_xr, fiber_channel, out_dir, fov, save_csv=False,
+                                      **kwargs)
+            fiber_object_table.append(subtable)
+
+            # update progress bar
+            fibseg_progress.update(1)
 
     fiber_object_table = pd.concat(fiber_object_table)
 

src/ark/utils/data_utils.py

@@ -327,10 +327,10 @@ def generate_and_save_cell_cluster_masks(
     )
 
     # create the pixel cluster masks across each fov
-    with tqdm(
-        total=len(fovs), desc="Cell Cluster Mask Generation", unit="FOVs"
-    ) as pbar:
+    with tqdm(total=len(fovs), desc="Cell Cluster Mask Generation", unit="FOVs") as pbar:
         for fov in fovs:
+            pbar.set_postfix(FOV=fov)
+
             # generate the cell mask for the FOV
             cell_mask: np.ndarray = generate_cell_cluster_mask(
                 fov=fov, seg_dir=seg_dir, ccmd=ccmd, seg_suffix=seg_suffix
@@ -461,8 +461,11 @@ def generate_and_save_pixel_cluster_masks(fovs: List[str],
     """
 
     # create the pixel cluster masks across each fov
-    with tqdm(total=len(fovs), desc="Pixel Cluster Mask Generation") as pixel_mask_progress:
+    with tqdm(total=len(fovs), desc="Pixel Cluster Mask Generation", unit="FOVs") \
+            as pixel_mask_progress:
         for fov in fovs:
+            pixel_mask_progress.set_postfix(FOV=fov)
+
             # define the path to provided channel file in the fov dir, used to calculate dimensions
             chan_file_path = os.path.join(fov, chan_file)
 
@@ -532,11 +535,12 @@ def generate_and_save_neighborhood_cluster_masks(
     )
 
     # create the neighborhood cluster masks across each fov
-    with tqdm(
-        total=len(fovs), desc="Neighborhood Cluster Mask Generation"
-    ) as neigh_mask_progress:
+    with tqdm(total=len(fovs), desc="Neighborhood Cluster Mask Generation", unit="FOVs") \
+            as neigh_mask_progress:
         # generate the mask for each FOV
         for fov in fovs:
+            neigh_mask_progress.set_postfix(FOV=fov)
+
             # load in the label map for the FOV
             label_map = load_utils.load_imgs_from_dir(
                 seg_dir,

src/ark/utils/deepcell_service_utils.py

@@ -296,13 +296,17 @@ def run_deepcell_direct(input_dir, output_dir, host='https://deepcell.org',
             }
         ).json()
         if redis_response['value'][0] == 'done':
+            # make sure progress bar shows 100%
+            pbar_next = int(redis_response['value'][1])
+            progress_bar.update(max(pbar_next - pbar_last, 0))
             break
 
         # update progress bar here
         if redis_response['value'][0] == 'waiting':
             pbar_next = int(redis_response['value'][1])
-            progress_bar.update(max(pbar_next - pbar_last, 0))
-            pbar_last = pbar_next
+            if pbar_next > pbar_last:
+                progress_bar.update(max(pbar_next - pbar_last, 0))
+                pbar_last = pbar_next
 
         if redis_response['value'][0] not in ['done', 'waiting', 'new']:
             print(redis_response['value'])

src/ark/utils/plot_utils.py

@@ -686,6 +686,8 @@ def save_colored_masks(
 
     with tqdm(total=len(fovs), desc="Saving colored masks", unit="FOVs") as pbar:
         for fov in fovs:
+            pbar.set_postfix(FOV=fov, refresh=False)
+
             mask: xr.DataArray = load_utils.load_imgs_from_dir(
                 data_dir=mask_dir,
                 files=[f"{fov}_{cluster_type}_mask.tiff"],
@@ -705,5 +707,4 @@ def save_colored_masks(
                 fname=save_dir / f"{fov}_{cluster_type}_mask_colored.tiff",
                 data=colored_mask,)
 
-            pbar.set_postfix(FOV=fov, refresh=False)
             pbar.update(1)