From 84ae124840eb989ee18a56aecd5f1e3e2b5c1065 Mon Sep 17 00:00:00 2001 From: csowers Date: Mon, 31 Jul 2023 11:46:58 -0700 Subject: [PATCH 1/6] move FOV update to top of function --- src/ark/analysis/cell_neighborhood_stats.py | 6 ++++-- src/ark/analysis/neighborhood_analysis.py | 3 ++- src/ark/analysis/spatial_analysis_utils.py | 3 ++- src/ark/utils/plot_utils.py | 3 ++- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/ark/analysis/cell_neighborhood_stats.py b/src/ark/analysis/cell_neighborhood_stats.py index 8e736a6f5..d6940fb65 100644 --- a/src/ark/analysis/cell_neighborhood_stats.py +++ b/src/ark/analysis/cell_neighborhood_stats.py @@ -51,6 +51,8 @@ def compute_neighborhood_diversity(neighborhood_mat, cell_type_col): fov_list = np.unique(neighborhood_mat[settings.FOV_ID]) with tqdm(total=len(fov_list), desc="Calculate Neighborhood Diversity") as diversity_progress: for fov in fov_list: + diversity_progress.set_postfix(FOV=fov) + fov_neighborhoods = neighborhood_mat[neighborhood_mat[settings.FOV_ID] == fov] diversity_scores = [] @@ -72,7 +74,6 @@ def compute_neighborhood_diversity(neighborhood_mat, cell_type_col): }) diversity_data.append(fov_data) - diversity_progress.set_postfix(FOV=fov) diversity_progress.update(1) # dataframe containing all fovs @@ -218,6 +219,8 @@ def generate_cell_distance_analysis( cell_dists = [] with tqdm(total=len(fov_list), desc="Calculate Average Distances") as distance_progress: for fov in fov_list: + distance_progress.set_postfix(FOV=fov) + fov_cell_table = cell_table[cell_table[fov_col] == fov] fov_dist_xr = xr.load_dataarray(os.path.join(dist_mat_dir, str(fov) + '_dist_mat.xr')) @@ -231,7 +234,6 @@ def generate_cell_distance_analysis( fov_cell_dists.insert(2, cell_type_col, fov_cell_table[cell_type_col]) cell_dists.append(fov_cell_dists) - distance_progress.set_postfix(FOV=fov) distance_progress.update(1) # combine data for all fovs and save to csv diff --git a/src/ark/analysis/neighborhood_analysis.py b/src/ark/analysis/neighborhood_analysis.py index ac5219dfc..d627e13d0 100644 --- a/src/ark/analysis/neighborhood_analysis.py +++ b/src/ark/analysis/neighborhood_analysis.py @@ -79,6 +79,8 @@ def create_neighborhood_matrix(all_data, dist_mat_dir, included_fovs=None, distl with tqdm(total=len(included_fovs), desc="Neighbors Matrix Generation") \ as neighbor_mat_progress: for fov in included_fovs: + neighbor_mat_progress.set_postfix(FOV=fov) + # Subsetting expression matrix to only include patients with correct fov label current_fov_idx = all_neighborhood_data.loc[:, fov_col] == fov current_fov_neighborhood_data = all_neighborhood_data[current_fov_idx] @@ -100,7 +102,6 @@ def create_neighborhood_matrix(all_data, dist_mat_dir, included_fovs=None, distl cell_neighbor_freqs.loc[current_fov_neighborhood_data.index, fov_cluster_names]\ = freqs - neighbor_mat_progress.set_postfix(FOV=fov) neighbor_mat_progress.update(1) # Remove cells that have no neighbors within the distlim diff --git a/src/ark/analysis/spatial_analysis_utils.py b/src/ark/analysis/spatial_analysis_utils.py index 5023a0cc0..4ea4d8ae6 100644 --- a/src/ark/analysis/spatial_analysis_utils.py +++ b/src/ark/analysis/spatial_analysis_utils.py @@ -39,6 +39,8 @@ def calc_dist_matrix(label_dir, save_path, prefix='_whole_cell'): # iterate for each fov with tqdm(total=len(fov_files), desc="Distance Matrix Generation") as dist_mat_progress: for fov_file in fov_files: + dist_mat_progress.set_postfix(FOV=fov_name) + # retrieve the fov name fov_name = fov_file.replace(prefix + '.tiff', '') @@ -66,7 +68,6 @@ def calc_dist_matrix(label_dir, save_path, prefix='_whole_cell'): format='NETCDF3_64BIT' ) - dist_mat_progress.set_postfix(FOV=fov_name) dist_mat_progress.update(1) diff --git a/src/ark/utils/plot_utils.py b/src/ark/utils/plot_utils.py index a31120161..319d3f4c1 100644 --- a/src/ark/utils/plot_utils.py +++ b/src/ark/utils/plot_utils.py @@ -686,6 +686,8 @@ def save_colored_masks( with tqdm(total=len(fovs), desc="Saving colored masks", unit="FOVs") as pbar: for fov in fovs: + pbar.set_postfix(FOV=fov, refresh=False) + mask: xr.DataArray = load_utils.load_imgs_from_dir( data_dir=mask_dir, files=[f"{fov}_{cluster_type}_mask.tiff"], @@ -705,5 +707,4 @@ def save_colored_masks( fname=save_dir / f"{fov}_{cluster_type}_mask_colored.tiff", data=colored_mask,) - pbar.set_postfix(FOV=fov, refresh=False) pbar.update(1) From 51e3eeed468c524149bd1f0a69e5350eece35212 Mon Sep 17 00:00:00 2001 From: csowers Date: Mon, 31 Jul 2023 14:27:24 -0700 Subject: [PATCH 2/6] add FOV name if missing --- src/ark/analysis/spatial_enrichment.py | 4 ++++ src/ark/utils/data_utils.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/src/ark/analysis/spatial_enrichment.py b/src/ark/analysis/spatial_enrichment.py index 61cef89e1..472e3ed93 100644 --- a/src/ark/analysis/spatial_enrichment.py +++ b/src/ark/analysis/spatial_enrichment.py @@ -72,6 +72,8 @@ def generate_channel_spatial_enrichment_stats(label_dir, dist_mat_dir, marker_th with tqdm(total=len(all_label_fovs), desc="Channel Spatial Enrichment") as chan_progress: for fov_name, label_file in zip(all_label_fovs, all_label_names): + chan_progress.set_postfix(FOV=fov_name) + label_maps = load_utils.load_imgs_from_dir(label_dir, files=[label_file], xr_channel_names=[xr_channel_name], trim_suffix=suffix) @@ -294,6 +296,8 @@ def generate_cluster_spatial_enrichment_stats(label_dir, dist_mat_dir, all_data, with tqdm(total=len(all_label_fovs), desc="Cluster Spatial Enrichment") as clust_progress: for fov_name, label_file in zip(all_label_fovs, all_label_names): + clust_progress.set_postfix(FOV=fov_name) + label_maps = load_utils.load_imgs_from_dir(label_dir, files=[label_file], xr_channel_names=[xr_channel_name], trim_suffix=suffix) diff --git a/src/ark/utils/data_utils.py b/src/ark/utils/data_utils.py index 240c91e00..7866d37f1 100644 --- a/src/ark/utils/data_utils.py +++ b/src/ark/utils/data_utils.py @@ -331,6 +331,8 @@ def generate_and_save_cell_cluster_masks( total=len(fovs), desc="Cell Cluster Mask Generation", unit="FOVs" ) as pbar: for fov in fovs: + pbar.set_postfix(FOV=fov) + # generate the cell mask for the FOV cell_mask: np.ndarray = generate_cell_cluster_mask( fov=fov, seg_dir=seg_dir, ccmd=ccmd, seg_suffix=seg_suffix @@ -463,6 +465,8 @@ def generate_and_save_pixel_cluster_masks(fovs: List[str], # create the pixel cluster masks across each fov with tqdm(total=len(fovs), desc="Pixel Cluster Mask Generation") as pixel_mask_progress: for fov in fovs: + pixel_mask_progress.set_postfix(FOV=fov) + # define the path to provided channel file in the fov dir, used to calculate dimensions chan_file_path = os.path.join(fov, chan_file) @@ -537,6 +541,8 @@ def generate_and_save_neighborhood_cluster_masks( ) as neigh_mask_progress: # generate the mask for each FOV for fov in fovs: + neigh_mask_progress.set_postfix(FOV=fov) + # load in the label map for the FOV label_map = load_utils.load_imgs_from_dir( seg_dir, From 03fef06014161fe85515557b771bb22859459968 Mon Sep 17 00:00:00 2001 From: csowers Date: Mon, 31 Jul 2023 14:39:35 -0700 Subject: [PATCH 3/6] add progress bar to fiber segmentation --- src/ark/segmentation/fiber_segmentation.py | 23 ++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/ark/segmentation/fiber_segmentation.py b/src/ark/segmentation/fiber_segmentation.py index df3bcfc9e..50f1aec22 100644 --- a/src/ark/segmentation/fiber_segmentation.py +++ b/src/ark/segmentation/fiber_segmentation.py @@ -16,6 +16,7 @@ from skimage.measure import regionprops_table from skimage.morphology import remove_small_objects from skimage.segmentation import watershed +from tqdm.notebook import tqdm from ark import settings from ark.utils.plot_utils import set_minimum_color_for_colormap @@ -176,14 +177,20 @@ def run_fiber_segmentation(data_dir, fiber_channel, out_dir, img_sub_folder=None fiber_object_table = [] - for fov in fovs: - print(f'Processing FOV: {fov}') - subset_xr = load_utils.load_imgs_from_tree( - data_dir, img_sub_folder, fovs=fov, channels=[fiber_channel] - ) - subtable = segment_fibers(subset_xr, fiber_channel, out_dir, fov, save_csv=False, - **kwargs) - fiber_object_table.append(subtable) + with tqdm(total=len(fovs), desc="Fiber Segmentation") as fibseg_progress: + for fov in fovs: + fibseg_progress.set_postfix(FOV=fov) + + subset_xr = load_utils.load_imgs_from_tree( + data_dir, img_sub_folder, fovs=fov, channels=[fiber_channel] + ) + # run fiber segmentation on the FOV + subtable = segment_fibers(subset_xr, fiber_channel, out_dir, fov, save_csv=False, + **kwargs) + fiber_object_table.append(subtable) + + # update progress bar + fibseg_progress.update(1) fiber_object_table = pd.concat(fiber_object_table) From 3b71fc1da51a6e6647bfc1e09cedd2a3e73d9ef9 Mon Sep 17 00:00:00 2001 From: csowers Date: Mon, 31 Jul 2023 17:09:01 -0700 Subject: [PATCH 4/6] arg name fix --- src/ark/analysis/spatial_analysis_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ark/analysis/spatial_analysis_utils.py b/src/ark/analysis/spatial_analysis_utils.py index 4ea4d8ae6..443b7ebb8 100644 --- a/src/ark/analysis/spatial_analysis_utils.py +++ b/src/ark/analysis/spatial_analysis_utils.py @@ -39,7 +39,7 @@ def calc_dist_matrix(label_dir, save_path, prefix='_whole_cell'): # iterate for each fov with tqdm(total=len(fov_files), desc="Distance Matrix Generation") as dist_mat_progress: for fov_file in fov_files: - dist_mat_progress.set_postfix(FOV=fov_name) + dist_mat_progress.set_postfix(FOV=fov_file) # retrieve the fov name fov_name = fov_file.replace(prefix + '.tiff', '') From 3fab6691acd991455709cf2e77114296135458cd Mon Sep 17 00:00:00 2001 From: csowers Date: Tue, 1 Aug 2023 13:31:04 -0700 Subject: [PATCH 5/6] fix progress bar for deepcell upload --- src/ark/utils/deepcell_service_utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ark/utils/deepcell_service_utils.py b/src/ark/utils/deepcell_service_utils.py index d5bd8b2b8..7a4ae2d25 100644 --- a/src/ark/utils/deepcell_service_utils.py +++ b/src/ark/utils/deepcell_service_utils.py @@ -296,13 +296,17 @@ def run_deepcell_direct(input_dir, output_dir, host='https://deepcell.org', } ).json() if redis_response['value'][0] == 'done': + # make sure progress bar shows 100% + pbar_next = int(redis_response['value'][1]) + progress_bar.update(max(pbar_next - pbar_last, 0)) break # update progress bar here if redis_response['value'][0] == 'waiting': pbar_next = int(redis_response['value'][1]) - progress_bar.update(max(pbar_next - pbar_last, 0)) - pbar_last = pbar_next + if pbar_next > pbar_last: + progress_bar.update(max(pbar_next - pbar_last, 0)) + pbar_last = pbar_next if redis_response['value'][0] not in ['done', 'waiting', 'new']: print(redis_response['value']) From b649695986b1d867996b0f5225979bfa9ff79251 Mon Sep 17 00:00:00 2001 From: csowers Date: Tue, 1 Aug 2023 15:34:11 -0700 Subject: [PATCH 6/6] units & imports --- src/ark/analysis/cell_neighborhood_stats.py | 6 ++++-- src/ark/analysis/neighborhood_analysis.py | 2 +- src/ark/analysis/spatial_analysis_utils.py | 3 ++- src/ark/analysis/spatial_enrichment.py | 6 ++++-- src/ark/segmentation/fiber_segmentation.py | 5 +++-- src/ark/utils/data_utils.py | 12 +++++------- 6 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/ark/analysis/cell_neighborhood_stats.py b/src/ark/analysis/cell_neighborhood_stats.py index d6940fb65..c82ad15d6 100644 --- a/src/ark/analysis/cell_neighborhood_stats.py +++ b/src/ark/analysis/cell_neighborhood_stats.py @@ -49,7 +49,8 @@ def compute_neighborhood_diversity(neighborhood_mat, cell_type_col): diversity_data = [] fov_list = np.unique(neighborhood_mat[settings.FOV_ID]) - with tqdm(total=len(fov_list), desc="Calculate Neighborhood Diversity") as diversity_progress: + with tqdm(total=len(fov_list), desc="Calculate Neighborhood Diversity", unit="FOVs") \ + as diversity_progress: for fov in fov_list: diversity_progress.set_postfix(FOV=fov) @@ -217,7 +218,8 @@ def generate_cell_distance_analysis( fov_list = np.unique(cell_table[fov_col]) cell_dists = [] - with tqdm(total=len(fov_list), desc="Calculate Average Distances") as distance_progress: + with tqdm(total=len(fov_list), desc="Calculate Average Distances", unit="FOVs") \ + as distance_progress: for fov in fov_list: distance_progress.set_postfix(FOV=fov) diff --git a/src/ark/analysis/neighborhood_analysis.py b/src/ark/analysis/neighborhood_analysis.py index d627e13d0..e8d60a4eb 100644 --- a/src/ark/analysis/neighborhood_analysis.py +++ b/src/ark/analysis/neighborhood_analysis.py @@ -76,7 +76,7 @@ def create_neighborhood_matrix(all_data, dist_mat_dir, included_fovs=None, distl cell_neighbor_freqs = cell_neighbor_counts.copy(deep=True) - with tqdm(total=len(included_fovs), desc="Neighbors Matrix Generation") \ + with tqdm(total=len(included_fovs), desc="Neighbors Matrix Generation", unit="FOVs") \ as neighbor_mat_progress: for fov in included_fovs: neighbor_mat_progress.set_postfix(FOV=fov) diff --git a/src/ark/analysis/spatial_analysis_utils.py b/src/ark/analysis/spatial_analysis_utils.py index 443b7ebb8..21e402d84 100644 --- a/src/ark/analysis/spatial_analysis_utils.py +++ b/src/ark/analysis/spatial_analysis_utils.py @@ -37,7 +37,8 @@ def calc_dist_matrix(label_dir, save_path, prefix='_whole_cell'): fov_files = io_utils.list_files(label_dir, substrs=prefix + '.tiff') # iterate for each fov - with tqdm(total=len(fov_files), desc="Distance Matrix Generation") as dist_mat_progress: + with tqdm(total=len(fov_files), desc="Distance Matrix Generation", unit="FOVs") \ + as dist_mat_progress: for fov_file in fov_files: dist_mat_progress.set_postfix(FOV=fov_file) diff --git a/src/ark/analysis/spatial_enrichment.py b/src/ark/analysis/spatial_enrichment.py index 472e3ed93..14986ed81 100644 --- a/src/ark/analysis/spatial_enrichment.py +++ b/src/ark/analysis/spatial_enrichment.py @@ -70,7 +70,8 @@ def generate_channel_spatial_enrichment_stats(label_dir, dist_mat_dir, marker_th values = [] stats_datasets = [] - with tqdm(total=len(all_label_fovs), desc="Channel Spatial Enrichment") as chan_progress: + with tqdm(total=len(all_label_fovs), desc="Channel Spatial Enrichment", unit="FOVs") \ + as chan_progress: for fov_name, label_file in zip(all_label_fovs, all_label_names): chan_progress.set_postfix(FOV=fov_name) @@ -294,7 +295,8 @@ def generate_cluster_spatial_enrichment_stats(label_dir, dist_mat_dir, all_data, values = [] stats_datasets = [] - with tqdm(total=len(all_label_fovs), desc="Cluster Spatial Enrichment") as clust_progress: + with tqdm(total=len(all_label_fovs), desc="Cluster Spatial Enrichment", unit="FOVs") \ + as clust_progress: for fov_name, label_file in zip(all_label_fovs, all_label_names): clust_progress.set_postfix(FOV=fov_name) diff --git a/src/ark/segmentation/fiber_segmentation.py b/src/ark/segmentation/fiber_segmentation.py index 50f1aec22..8fa7ff54c 100644 --- a/src/ark/segmentation/fiber_segmentation.py +++ b/src/ark/segmentation/fiber_segmentation.py @@ -16,7 +16,7 @@ from skimage.measure import regionprops_table from skimage.morphology import remove_small_objects from skimage.segmentation import watershed -from tqdm.notebook import tqdm +from tqdm.auto import tqdm from ark import settings from ark.utils.plot_utils import set_minimum_color_for_colormap @@ -177,7 +177,8 @@ def run_fiber_segmentation(data_dir, fiber_channel, out_dir, img_sub_folder=None fiber_object_table = [] - with tqdm(total=len(fovs), desc="Fiber Segmentation") as fibseg_progress: + with tqdm(total=len(fovs), desc="Fiber Segmentation", unit="FOVs") \ + as fibseg_progress: for fov in fovs: fibseg_progress.set_postfix(FOV=fov) diff --git a/src/ark/utils/data_utils.py b/src/ark/utils/data_utils.py index 7866d37f1..ccc4a6fec 100644 --- a/src/ark/utils/data_utils.py +++ b/src/ark/utils/data_utils.py @@ -327,9 +327,7 @@ def generate_and_save_cell_cluster_masks( ) # create the pixel cluster masks across each fov - with tqdm( - total=len(fovs), desc="Cell Cluster Mask Generation", unit="FOVs" - ) as pbar: + with tqdm(total=len(fovs), desc="Cell Cluster Mask Generation", unit="FOVs") as pbar: for fov in fovs: pbar.set_postfix(FOV=fov) @@ -463,7 +461,8 @@ def generate_and_save_pixel_cluster_masks(fovs: List[str], """ # create the pixel cluster masks across each fov - with tqdm(total=len(fovs), desc="Pixel Cluster Mask Generation") as pixel_mask_progress: + with tqdm(total=len(fovs), desc="Pixel Cluster Mask Generation", unit="FOVs") \ + as pixel_mask_progress: for fov in fovs: pixel_mask_progress.set_postfix(FOV=fov) @@ -536,9 +535,8 @@ def generate_and_save_neighborhood_cluster_masks( ) # create the neighborhood cluster masks across each fov - with tqdm( - total=len(fovs), desc="Neighborhood Cluster Mask Generation" - ) as neigh_mask_progress: + with tqdm(total=len(fovs), desc="Neighborhood Cluster Mask Generation", unit="FOVs") \ + as neigh_mask_progress: # generate the mask for each FOV for fov in fovs: neigh_mask_progress.set_postfix(FOV=fov)