Re-run for clean model output

jpkemp · Aug 5, 2024 · 23e50f2 · 23e50f2
1 parent 3c708cf
commit 23e50f2
Show file tree

Hide file tree

Showing 48 changed files with 763 additions and 3,714 deletions.
diff --git a/README.md b/README.md
@@ -7,6 +7,10 @@ The file 'aci.ipynb' explores the effect of parameter selection on the Acoustic
 
 The file 'within_study_test.ipynb' explores the effect of parameter selection on four common acoustic indices: Acoustic Complexity Index, Acoustic Diversity Index, Acoustic Evenness Index, and Bioacoustic Index on two datasets, one from Carara National Park in Costa Rica and one from Big Vicky's Reef in Australia ([[1]](#1),[[2]](#2),[[3]](#3) ).
 
+The file 'posterior_checks'.ipynb presents the denisty overlay and scatter average posteriro checks for the within-study test.
+
+The file 'frequency_impact.ipynb' explores the effect of frequency on the interaction between simulated calls and FFT window parameter selection.
+
 ## Contact
 Please contact me with any questions
 <a href=https://au.linkedin.com/in/james-kemp-11874a93><img src=https://blog-assets.hootsuite.com/wp-content/uploads/2018/09/In-2C-54px-R.png

diff --git a/frequency_impact.ipynb b/frequency_impact.ipynb
diff --git a/output/ACI_call frequency effect_rate_plot.png b/output/ACI_call frequency effect_rate_plot.png
diff --git a/output/ACI_call interval effect_rate_plot.png b/output/ACI_call interval effect_rate_plot.png
diff --git a/output/ADI_call frequency effect_rate_plot.png b/output/ADI_call frequency effect_rate_plot.png
diff --git a/output/ADI_call interval effect_rate_plot.png b/output/ADI_call interval effect_rate_plot.png
diff --git a/output/AEI_call frequency effect_rate_plot.png b/output/AEI_call frequency effect_rate_plot.png
diff --git a/output/AEI_call interval effect_rate_plot.png b/output/AEI_call interval effect_rate_plot.png
diff --git a/output/BIO_call frequency effect_rate_plot.png b/output/BIO_call frequency effect_rate_plot.png
diff --git a/output/BIO_call interval effect_rate_plot.png b/output/BIO_call interval effect_rate_plot.png
diff --git a/...Hour_x_Window_conditional_effects_for_ACI_over_broadband_frequencies_Python.png b/...Hour_x_Window_conditional_effects_for_ACI_over_broadband_frequencies_Python.png
diff --git a/..._x_Window_conditional_effects_for_ACI_over_filtered_fish_frequencies_Python.png b/..._x_Window_conditional_effects_for_ACI_over_filtered_fish_frequencies_Python.png
diff --git a/..._Window_conditional_effects_for_ACI_over_filtered_shrimp_frequencies_Python.png b/..._Window_conditional_effects_for_ACI_over_filtered_shrimp_frequencies_Python.png
diff --git a/output/Hour_x_Window_conditional_effects_for_ACI_over_fish_frequencies_Python.png b/output/Hour_x_Window_conditional_effects_for_ACI_over_fish_frequencies_Python.png
diff --git a/...ut/Hour_x_Window_conditional_effects_for_ACI_over_shrimp_frequencies_Python.png b/...ut/Hour_x_Window_conditional_effects_for_ACI_over_shrimp_frequencies_Python.png
diff --git a/...Hour_x_Window_conditional_effects_for_ADI_over_broadband_frequencies_Python.png b/...Hour_x_Window_conditional_effects_for_ADI_over_broadband_frequencies_Python.png
diff --git a/..._x_Window_conditional_effects_for_ADI_over_filtered_fish_frequencies_Python.png b/..._x_Window_conditional_effects_for_ADI_over_filtered_fish_frequencies_Python.png
diff --git a/..._Window_conditional_effects_for_ADI_over_filtered_shrimp_frequencies_Python.png b/..._Window_conditional_effects_for_ADI_over_filtered_shrimp_frequencies_Python.png
diff --git a/output/Hour_x_Window_conditional_effects_for_ADI_over_fish_frequencies_Python.png b/output/Hour_x_Window_conditional_effects_for_ADI_over_fish_frequencies_Python.png
diff --git a/...ut/Hour_x_Window_conditional_effects_for_ADI_over_shrimp_frequencies_Python.png b/...ut/Hour_x_Window_conditional_effects_for_ADI_over_shrimp_frequencies_Python.png
diff --git a/...Hour_x_Window_conditional_effects_for_AEI_over_broadband_frequencies_Python.png b/...Hour_x_Window_conditional_effects_for_AEI_over_broadband_frequencies_Python.png
diff --git a/..._x_Window_conditional_effects_for_AEI_over_filtered_fish_frequencies_Python.png b/..._x_Window_conditional_effects_for_AEI_over_filtered_fish_frequencies_Python.png
diff --git a/..._Window_conditional_effects_for_AEI_over_filtered_shrimp_frequencies_Python.png b/..._Window_conditional_effects_for_AEI_over_filtered_shrimp_frequencies_Python.png
diff --git a/output/Hour_x_Window_conditional_effects_for_AEI_over_fish_frequencies_Python.png b/output/Hour_x_Window_conditional_effects_for_AEI_over_fish_frequencies_Python.png
diff --git a/...ut/Hour_x_Window_conditional_effects_for_AEI_over_shrimp_frequencies_Python.png b/...ut/Hour_x_Window_conditional_effects_for_AEI_over_shrimp_frequencies_Python.png
diff --git a/...Hour_x_Window_conditional_effects_for_BIO_over_broadband_frequencies_Python.png b/...Hour_x_Window_conditional_effects_for_BIO_over_broadband_frequencies_Python.png
diff --git a/..._x_Window_conditional_effects_for_BIO_over_filtered_fish_frequencies_Python.png b/..._x_Window_conditional_effects_for_BIO_over_filtered_fish_frequencies_Python.png
diff --git a/..._Window_conditional_effects_for_BIO_over_filtered_shrimp_frequencies_Python.png b/..._Window_conditional_effects_for_BIO_over_filtered_shrimp_frequencies_Python.png
diff --git a/output/Hour_x_Window_conditional_effects_for_BIO_over_fish_frequencies_Python.png b/output/Hour_x_Window_conditional_effects_for_BIO_over_fish_frequencies_Python.png
diff --git a/...ut/Hour_x_Window_conditional_effects_for_BIO_over_shrimp_frequencies_Python.png b/...ut/Hour_x_Window_conditional_effects_for_BIO_over_shrimp_frequencies_Python.png
diff --git a/...Site_x_Window_conditional_effects_for_ACI_over_broadband_frequencies_Python.png b/...Site_x_Window_conditional_effects_for_ACI_over_broadband_frequencies_Python.png
diff --git a/...Site_x_Window_conditional_effects_for_ADI_over_broadband_frequencies_Python.png b/...Site_x_Window_conditional_effects_for_ADI_over_broadband_frequencies_Python.png
diff --git a/...Site_x_Window_conditional_effects_for_AEI_over_broadband_frequencies_Python.png b/...Site_x_Window_conditional_effects_for_AEI_over_broadband_frequencies_Python.png
diff --git a/...Site_x_Window_conditional_effects_for_BIO_over_broadband_frequencies_Python.png b/...Site_x_Window_conditional_effects_for_BIO_over_broadband_frequencies_Python.png
diff --git a/output/combined_Hour_broadband_False.png b/output/combined_Hour_broadband_False.png
diff --git a/output/combined_Hour_fish_False.png b/output/combined_Hour_fish_False.png
diff --git a/output/combined_Hour_fish_True.png b/output/combined_Hour_fish_True.png
diff --git a/output/combined_Hour_shrimp_False.png b/output/combined_Hour_shrimp_False.png
diff --git a/output/combined_Hour_shrimp_True.png b/output/combined_Hour_shrimp_True.png
diff --git a/output/combined_Site_broadband_False.png b/output/combined_Site_broadband_False.png
diff --git a/output/combined_fish_ADI.png b/output/combined_fish_ADI.png
diff --git a/output/combined_frequency_ACI_BIO.png b/output/combined_frequency_ACI_BIO.png
diff --git a/output/combined_shrimp_AEI.png b/output/combined_shrimp_AEI.png
diff --git a/posterior_checks.ipynb b/posterior_checks.ipynb
diff --git a/tools/plot_combiner.py b/tools/plot_combiner.py
@@ -10,16 +10,26 @@
 
 class PlotCombiner:
     '''graph combining functions'''
-    @classmethod
-    def open_figure(cls, filename, discard_lgd=False):
+    def __init__(self) -> None:
+        self.fig_bufs = []
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        for buf in self.fig_bufs:
+            buf.close()
+
+    def open_figure(self, filename, discard_lgd=False):
         fig = pickle.load(open(filename, 'rb'))
         lgd = fig.get_axes()[0].get_legend()
         if discard_lgd:
             lgd.remove()
             buf = None
         else:
             buf = io.BytesIO()
-            cls.export_legend(lgd, buf)
+            self.fig_bufs.append(buf)
+            self.export_legend(lgd, buf)
             lgd.remove()
 
         ax = fig.gca()
@@ -55,7 +65,8 @@ def combine_figures(cls, plots, rows=2, imgs_per_row=None, spacing=20, include_l
                 images.append(Image.open(buf))
                 if lgd_buf: lgd = Image.open(lgd_buf)
 
-            mod = len(images) % imgs_per_row
+            n_imgs = len(images)
+            mod = n_imgs % imgs_per_row
             final = []
             subset = images
             if mod:
@@ -67,10 +78,17 @@ def combine_figures(cls, plots, rows=2, imgs_per_row=None, spacing=20, include_l
             if include_lgd:
                 lgd_width += lgd.size[0] + spacing
 
-            total_width = ceil(sum(widths) / imgs_per_row) + lgd_width
-            max_height = ceil(max(heights) * rows) + (rows - 1) * (int(spacing))
+            max_width = 0
+            max_height = 0
+            for i in range(0, n_imgs, imgs_per_row):
+                row_width = sum(widths[i:i+imgs_per_row])
+                max_width = max(max_width, row_width)
+                max_height += max(heights[i:i+imgs_per_row])
+
+            total_width = max_width + lgd_width
+            total_height = ceil(max(heights) * rows) + (rows - 1) * (int(spacing))
 
-            new_im = Image.new('RGB', (total_width, max_height), color="white")
+            new_im = Image.new('RGB', (total_width, total_height), color="white")
 
             x_offset = 0
             y_offset = 0
@@ -91,7 +109,7 @@ def combine_figures(cls, plots, rows=2, imgs_per_row=None, spacing=20, include_l
 
             if include_lgd:
                 x_offset = total_width - lgd_width - spacing
-                y_offset = ceil((max_height / 2) - (lgd.size[1] / 2)) + 10
+                y_offset = ceil((total_height / 2) - (lgd.size[1] / 2)) + 10
                 new_im.paste(lgd, (x_offset,y_offset))
 
 
@@ -119,12 +137,13 @@ def get_figures_from_folders(cls, folder_path, only_include=None, sort_by=None):
     @classmethod
     def combine_plots(cls, filenames, output_folder, output_file_notation, max_figs_per_row=2):
         '''wrapper for graph combining process'''
-        figs = [cls.open_figure(x, bool(i)) for i, x in enumerate(filenames)]
-        n_figs = len(figs)
-        rows = ceil(n_figs / max_figs_per_row)
-        final_fig = cls.combine_figures(figs, rows, max_figs_per_row)
-        output_filename = str(output_folder) + f"/combined_{output_file_notation}.png"
-        final_fig.save(output_filename)
+        with PlotCombiner() as combiner:
+            figs = [combiner.open_figure(x, bool(i)) for i, x in enumerate(filenames)]
+            n_figs = len(figs)
+            rows = ceil(n_figs / max_figs_per_row)
+            final_fig = cls.combine_figures(figs, rows, max_figs_per_row)
+            output_filename = str(output_folder) + f"/combined_{output_file_notation}.png"
+            final_fig.save(output_filename)
 
 if __name__ == "__main__":
     def inc(tgt:str, band:str, flt:bool, x):
@@ -139,7 +158,7 @@ def inc(tgt:str, band:str, flt:bool, x):
 
         return True
 
-    combiner = PlotCombiner()
+    combiner = PlotCombiner
     targets = ["Site", "Hour"]
     bands = ["broadband", "shrimp", "fish"]
     filtered = [True, False]
@@ -148,6 +167,18 @@ def inc(tgt:str, band:str, flt:bool, x):
         only_inc = partial(inc, target, band, fltr)
         filenames = combiner.get_figures_from_folders("output", only_inc, sorting)
         if not filenames: continue
-        combiner.combine_plots("output", f"{target}_{band}_{fltr}")
+        combiner.combine_plots(filenames, "output", f"{target}_{band}_{fltr}")
+
+
+    filenames = [f"output/{x}_call frequency effect_rate_plot.pkl" for x in ["ACI", "BIO"]]
+    combiner.combine_plots(filenames, "output", f"frequency_ACI_BIO")
+
+    filenames = ["output/Hour_x_Window_conditional_effects_for_AEI_over_filtered_shrimp_frequencies_Python.pkl",
+                "output/Hour_x_Window_conditional_effects_for_AEI_over_shrimp_frequencies_Python.pkl"]
+    combiner.combine_plots(filenames, "output", f"shrimp_AEI")
+
+    filenames = ["output/Hour_x_Window_conditional_effects_for_ADI_over_filtered_fish_frequencies_Python.pkl",
+                "output/Hour_x_Window_conditional_effects_for_ADI_over_fish_frequencies_Python.pkl"]
+    combiner.combine_plots(filenames, "output", f"fish_ADI")
 
 
diff --git a/tools/plots.py b/tools/plots.py
@@ -58,7 +58,7 @@ def plot_conditional_effects(r_link,
         with context():
             point_df = pandas2ri.rpy2py(points)
 
-        for var_id, group in point_df.groupby(effect_names[1]):
+        for var_id, group in point_df.groupby(effect_names[1], observed=False):
             x = group[effect_names[0]] #-  int(var_id)
             y = group['resp__']
             if unlog:
@@ -70,7 +70,7 @@ def plot_conditional_effects(r_link,
     ylabel = df.columns[2]
     legend_title = df.columns[1]
 
-    for var_id, group in df.groupby(effect_names[1]):
+    for var_id, group in df.groupby(effect_names[1], observed=False):
         x = group[effect_names[0]]
         var_line = group["estimate__"]
         upperlimit = group["upper__"]

diff --git a/tools/r_funcs.R b/tools/r_funcs.R
@@ -89,10 +89,6 @@ get_posterior_ratios <- function(model, num="12", den="0", cross="Hour") {
     dplyr::ungroup()
 }
 
-# calc_dist_same_ratio <- function(x, thresh=0.1) {
-#  sum(x > -thresh & x < thresh) / length(x)
-# }
-
 get_posterior_ratio_difference <- function(posterior_ratios, num="12", den="0", cross="Hour") {
     posterior_ratios |>
     dplyr::group_by((.draw)) |>

diff --git a/within_study_test.ipynb b/within_study_test.ipynb