diff --git a/Microfaune_Local_Score_Package_Tutorial.ipynb b/Microfaune_Local_Score_Package_Tutorial.ipynb index 022eeda..8dfb99a 100644 --- a/Microfaune_Local_Score_Package_Tutorial.ipynb +++ b/Microfaune_Local_Score_Package_Tutorial.ipynb @@ -2357,534 +2357,6 @@ "source": [ "manual_df_with_IoU" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Function that determines the overlap between human and automated labels with respect to the number of samples in the human label. Referred to as \"Catch\"" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[1. 0.7344 1. 1. 0.7588]\n" - ] - } - ], - "source": [ - "Catch_Array = clip_catch(automated_piha_df,manual_piha_df)\n", - "print(Catch_Array)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Function that determines the label-by-label \"Catch\" across multiple clips" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "20190622_210000.WAV\n", - "20190623_222000.WAV\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/jacob/Desktop/EngineersForExploration/Bioacoustics/Automated_Audio_Labelling_System_AID/microfaune_local_score.py:855: SettingWithCopyWarning: \n", - "A value is trying to be set on a copy of a slice from a DataFrame.\n", - "Try using .loc[row_indexer,col_indexer] = value instead\n", - "\n", - "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", - " clip_manual_df[\"Catch\"] = Catch_Array\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "BlackFacedAntbird1.wav\n", - "HowlerMonkey1.WAV\n", - "20190624_152000.WAV\n", - "ScreamingPiha2.wav\n" - ] - } - ], - "source": [ - "manual_df_with_catch = dataset_Catch(automated_df,manual_df)" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
FOLDERIN FILECHANNELCLIP LENGTHOFFSETDURATIONMANUAL IDCatch
0/home/jacob/Acoustic-Species-Identification/pa...20190622_210000.WAV060.0000000.0009.70011.0000
1/home/jacob/Acoustic-Species-Identification/pa...20190622_210000.WAV060.00000011.50016.67511.0000
2/home/jacob/Acoustic-Species-Identification/pa...20190622_210000.WAV060.00000029.50015.00010.7885
3/home/jacob/Acoustic-Species-Identification/pa...20190622_210000.WAV060.00000048.5251.00010.0000
4/home/jacob/Acoustic-Species-Identification/pa...20190622_210000.WAV060.00000050.5008.00010.8444
5/home/jacob/Acoustic-Species-Identification/pa...20190623_222000.WAV060.00000014.50012.50010.6778
6/home/jacob/Acoustic-Species-Identification/pa...BlackFacedAntbird1.wav031.2000000.0004.10010.6448
7/home/jacob/Acoustic-Species-Identification/pa...BlackFacedAntbird1.wav031.2000005.0001.50010.8951
8/home/jacob/Acoustic-Species-Identification/pa...BlackFacedAntbird1.wav031.2000009.8005.20010.4523
9/home/jacob/Acoustic-Species-Identification/pa...BlackFacedAntbird1.wav031.20000021.5005.50010.4737
10/home/jacob/Acoustic-Species-Identification/pa...HowlerMonkey1.WAV060.0000000.00010.90010.8562
11/home/jacob/Acoustic-Species-Identification/pa...HowlerMonkey1.WAV060.00000011.50013.00010.4616
12/home/jacob/Acoustic-Species-Identification/pa...HowlerMonkey1.WAV060.00000025.00034.90010.6903
13/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV060.0000001.2000.45010.0000
14/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV060.0000004.0000.30010.0000
15/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV060.0000007.4000.60010.4630
16/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV060.00000010.5000.80011.0000
17/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV060.00000020.8000.50010.0000
18/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV060.00000028.0000.40010.0000
19/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV060.00000034.0001.00010.0000
20/home/jacob/Acoustic-Species-Identification/pa...ScreamingPiha2.wav033.9330610.0005.37311.0000
21/home/jacob/Acoustic-Species-Identification/pa...ScreamingPiha2.wav033.93306110.5905.58510.7344
22/home/jacob/Acoustic-Species-Identification/pa...ScreamingPiha2.wav033.93306122.0004.00011.0000
23/home/jacob/Acoustic-Species-Identification/pa...ScreamingPiha2.wav033.93306127.2000.90011.0000
24/home/jacob/Acoustic-Species-Identification/pa...ScreamingPiha2.wav033.93306129.0004.50010.7588
\n", - "
" - ], - "text/plain": [ - " FOLDER IN FILE \\\n", - "0 /home/jacob/Acoustic-Species-Identification/pa... 20190622_210000.WAV \n", - "1 /home/jacob/Acoustic-Species-Identification/pa... 20190622_210000.WAV \n", - "2 /home/jacob/Acoustic-Species-Identification/pa... 20190622_210000.WAV \n", - "3 /home/jacob/Acoustic-Species-Identification/pa... 20190622_210000.WAV \n", - "4 /home/jacob/Acoustic-Species-Identification/pa... 20190622_210000.WAV \n", - "5 /home/jacob/Acoustic-Species-Identification/pa... 20190623_222000.WAV \n", - "6 /home/jacob/Acoustic-Species-Identification/pa... BlackFacedAntbird1.wav \n", - "7 /home/jacob/Acoustic-Species-Identification/pa... BlackFacedAntbird1.wav \n", - "8 /home/jacob/Acoustic-Species-Identification/pa... BlackFacedAntbird1.wav \n", - "9 /home/jacob/Acoustic-Species-Identification/pa... BlackFacedAntbird1.wav \n", - "10 /home/jacob/Acoustic-Species-Identification/pa... HowlerMonkey1.WAV \n", - "11 /home/jacob/Acoustic-Species-Identification/pa... HowlerMonkey1.WAV \n", - "12 /home/jacob/Acoustic-Species-Identification/pa... HowlerMonkey1.WAV \n", - "13 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", - "14 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", - "15 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", - "16 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", - "17 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", - "18 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", - "19 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", - "20 /home/jacob/Acoustic-Species-Identification/pa... ScreamingPiha2.wav \n", - "21 /home/jacob/Acoustic-Species-Identification/pa... ScreamingPiha2.wav \n", - "22 /home/jacob/Acoustic-Species-Identification/pa... ScreamingPiha2.wav \n", - "23 /home/jacob/Acoustic-Species-Identification/pa... ScreamingPiha2.wav \n", - "24 /home/jacob/Acoustic-Species-Identification/pa... ScreamingPiha2.wav \n", - "\n", - " CHANNEL CLIP LENGTH OFFSET DURATION MANUAL ID Catch \n", - "0 0 60.000000 0.000 9.700 1 1.0000 \n", - "1 0 60.000000 11.500 16.675 1 1.0000 \n", - "2 0 60.000000 29.500 15.000 1 0.7885 \n", - "3 0 60.000000 48.525 1.000 1 0.0000 \n", - "4 0 60.000000 50.500 8.000 1 0.8444 \n", - "5 0 60.000000 14.500 12.500 1 0.6778 \n", - "6 0 31.200000 0.000 4.100 1 0.6448 \n", - "7 0 31.200000 5.000 1.500 1 0.8951 \n", - "8 0 31.200000 9.800 5.200 1 0.4523 \n", - "9 0 31.200000 21.500 5.500 1 0.4737 \n", - "10 0 60.000000 0.000 10.900 1 0.8562 \n", - "11 0 60.000000 11.500 13.000 1 0.4616 \n", - "12 0 60.000000 25.000 34.900 1 0.6903 \n", - "13 0 60.000000 1.200 0.450 1 0.0000 \n", - "14 0 60.000000 4.000 0.300 1 0.0000 \n", - "15 0 60.000000 7.400 0.600 1 0.4630 \n", - "16 0 60.000000 10.500 0.800 1 1.0000 \n", - "17 0 60.000000 20.800 0.500 1 0.0000 \n", - "18 0 60.000000 28.000 0.400 1 0.0000 \n", - "19 0 60.000000 34.000 1.000 1 0.0000 \n", - "20 0 33.933061 0.000 5.373 1 1.0000 \n", - "21 0 33.933061 10.590 5.585 1 0.7344 \n", - "22 0 33.933061 22.000 4.000 1 1.0000 \n", - "23 0 33.933061 27.200 0.900 1 1.0000 \n", - "24 0 33.933061 29.000 4.500 1 0.7588 " - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "manual_df_with_catch" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "ename": "NameError", - "evalue": "name 'calc_local_scores_simpler' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mnew_automated_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcalc_local_scores_simpler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;31mNameError\u001b[0m: name 'calc_local_scores_simpler' is not defined" - ] - } - ], - "source": [ - "new_automated_df = calc_local_scores_simpler(path)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "new_automated_df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "local_score_visualization2(\"./TEST/ScreamingPiha2.wav\",automated_df = True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "stats_df2 = dataset_IoU_Statistics(new_automated_df,manual_df,threshold = 0.5)\n", - "stats_df2" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "global_stats_df = global_IoU_Statistics(stats_df2)\n", - "global_stats_df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "annotation_duration_statistics(new_automated_df)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/ScreamingPiha2_Manual_Labels.csv b/ScreamingPiha2_Manual_Labels.csv deleted file mode 100644 index 93804de..0000000 --- a/ScreamingPiha2_Manual_Labels.csv +++ /dev/null @@ -1,6 +0,0 @@ -FOLDER,IN FILE,CHANNEL,CLIP LENGTH,OFFSET,DURATION,MANUAL ID -/home/jacob/Acoustic-Species-Identification/passive-acoustic-biodiversity/BinaryBirdDet/TEST/,ScreamingPiha2.wav,0,33.9330612244898,0,5.373,1 -/home/jacob/Acoustic-Species-Identification/passive-acoustic-biodiversity/BinaryBirdDet/TEST/,ScreamingPiha2.wav,1,34.9330612244898,10.59,5.585,1 -/home/jacob/Acoustic-Species-Identification/passive-acoustic-biodiversity/BinaryBirdDet/TEST/,ScreamingPiha2.wav,2,35.9330612244898,22,4,1 -/home/jacob/Acoustic-Species-Identification/passive-acoustic-biodiversity/BinaryBirdDet/TEST/,ScreamingPiha2.wav,3,36.9330612244898,27.2,0.9,1 -/home/jacob/Acoustic-Species-Identification/passive-acoustic-biodiversity/BinaryBirdDet/TEST/,ScreamingPiha2.wav,4,37.9330612244898,29,4.5,1 diff --git a/microfaune_local_score.py b/microfaune_local_score.py index 0de553b..025d122 100644 --- a/microfaune_local_score.py +++ b/microfaune_local_score.py @@ -18,10 +18,6 @@ import math -# Another option would be to allow for curve smoothing on the local score array that is being passed in. This could come in -# the form of a high order polynomial fit or possibly testing out my curve smoothing algorithm that uses a bell-curved distribution to -# loop around and average each sample with its surrounding samples over many iterations. We could also play around with filtering. - # function that encapsulates many different isolation techniques to the dictionary isolation_parameters def isolate(local_scores, SIGNAL, SAMPLE_RATE, audio_dir, filename,isolation_parameters,manual_id = "bird"): @@ -35,12 +31,25 @@ def isolate(local_scores, SIGNAL, SAMPLE_RATE, audio_dir, filename,isolation_par isolation_df = steinberg_isolate(local_scores, SIGNAL, SAMPLE_RATE, audio_dir, filename,isolation_parameters, manual_id = "bird") elif isolation_parameters["technique"] == "stack": isolation_df = stack_isolate(local_scores, SIGNAL, SAMPLE_RATE, audio_dir, filename, isolation_parameters, manual_id = "bird") - # stack_isolate(local_scores, SIGNAL, SAMPLE_RATE, audio_dir, filename, isolation_parameters, manual_id = "bird"): return isolation_df def steinberg_isolate(local_scores, SIGNAL, SAMPLE_RATE, audio_dir, filename,isolation_parameters,manual_id = "bird"): + """ + Returns a dataframe of automated labels for the given audio clip. The automated labels determine intervals of bird noise as + determined by the local scores given by an RNNDetector. + + Args: + scores (list of floats) - Local scores of the audio clip as determined by RNNDetector. + SIGNAL (list of ints) - Samples from the audio clip. + SAMPLE_RATE (int) - Sampling rate of the audio clip, usually 44100. + audio_dir (string) - Directory of the audio clip. + filename (string) - Name of the audio clip file. + + Returns: + Dataframe of automated labels for the audio clip. + """ # calculate original duration old_duration = len(SIGNAL) / SAMPLE_RATE @@ -270,8 +279,21 @@ def stack_isolate(local_scores, SIGNAL, SAMPLE_RATE, audio_dir, filename, isolat # returning pandas dataframe from dictionary constructed with all of the annotations return pd.DataFrame.from_dict(entry) + ## Function that applies the moment to moment labeling system to a directory full of wav files. def generate_automated_labels(bird_dir, isolation_parameters, weight_path=None, Normalized_Sample_Rate = 44100): + """ + Function that applies the moment to moment labeling system to a directory full of wav files. + + Args: + bird_dir (string) - Directory with wav audio files. + weight_path (string) - File path of weights to be used by the RNNDetector for determining presence of bird sounds. + Normalized_Sample_Rate (int) - Sampling rate that the audio files should all be normalized to. + + Returns: + Dataframe of automated labels for the audio clips in bird_dir. + """ + # init detector # Use Default Microfaune Detector if weight_path is None: @@ -352,8 +374,24 @@ def annotation_duration_statistics(df): # returning the dictionary as a pandas dataframe return pd.DataFrame.from_dict([entry]) -# Function that produces graphs with the local score plot and spectrogram of an audio clip. Now integrated with Pandas so you can visualize human and automated annotations. + def local_line_graph(local_scores,clip_name, sample_rate,samples, automated_df=None, human_df=None,log_scale = False, save_fig = False): + """ + Function that produces graphs with the local score plot and spectrogram of an audio clip. Now integrated with Pandas so you can visualize human and automated annotations. + + Args: + local_scores (list of floats) - Local scores for the clip determined by the RNN. + clip_name (string) - Directory of the clip. + sample_rate (int) - Sample rate of the audio clip, usually 44100. + samples (list of ints) - Each of the samples from the audio clip. + automated_df (Dataframe) - Dataframe of automated labelling of the clip. + human_df (Dataframe) - Dataframe of human labelling of the clip. + log_scale (boolean) - Whether the axis for local scores should be logarithmically scaled on the plot. + save_fig (boolean) - Whether the clip should be saved in a directory as a png file. + + Returns: + None + """ # Calculating the length of the audio clip duration = samples.shape[0]/sample_rate # Calculating the number of local scores outputted by Microfaune @@ -412,11 +450,27 @@ def local_line_graph(local_scores,clip_name, sample_rate,samples, automated_df=N if save_fig: plt.savefig(clip_name + "_Local_Score_Graph.png") -# Wrapper function for the local_line_graph function for ease of use. # TODO rework function so that instead of generating the automated labels, it takes the automated_df as input # same as it does with the manual dataframe. + def local_score_visualization(clip_path,weight_path = None, human_df = None,automated_df = False, isolation_parameters = None,log_scale = False, save_fig = False): + """ + Wrapper function for the local_line_graph function for ease of use. Processes clip for local scores to be used for + the local_line_graph function. + + Args: + clip_path (string) - Directory of the clip. + weight_path (string) - Weights to be used for RNNDetector. + human_df (Dataframe) - Dataframe of human labels for the audio clip. + automated_df (Dataframe) - Whether the audio clip should be labelled by the isolate function and subsequently plotted. + log_scale (boolean) - Whether the axis for local scores should be logarithmically scaled on the plot. + save_fig (boolean) - Whether the plots should be saved in a directory as a png file. + + Returns: + None + """ + # Loading in the clip with Microfaune's built-in loading function SAMPLE_RATE, SIGNAL = audio.load_wav(clip_path) # downsample the audio if the sample rate > 44.1 kHz @@ -455,7 +509,19 @@ def local_score_visualization(clip_path,weight_path = None, human_df = None,auto def bird_label_scores(automated_df,human_df,plot_fig = False, save_fig = False): - + """ + Function to generate a dataframe with statistics relating to the efficiency of the automated label compared to the human label. + These statistics include true positive, false positive, false negative, true negative, union, precision, recall, F1, and Global IoU. + + Args: + automated_df (Dataframe) - Dataframe of automated labels for one clip + human_df (Dataframe) - Dataframe of human labels for one clip. + plot_fig (boolean) - Whether or not the efficiency statistics should be displayed. + save_fig (boolean) - Whether or not the plot should be saved within a file. + + Returns: + Dataframe with statistics comparing the automated and human labeling. + """ duration = automated_df["CLIP LENGTH"].to_list()[0] SAMPLE_RATE = automated_df["SAMPLE RATE"].to_list()[0] # Initializing two arrays that will represent the human labels and automated labels with respect to @@ -565,9 +631,23 @@ def bird_label_scores(automated_df,human_df,plot_fig = False, save_fig = False): return pd.DataFrame(entry,index=[0]) -# Function that will allow users to easily pass in two dataframes, and it will output statistics on them # Will have to adjust the isolate function so that it adds a sampling rate onto the dataframes. def automated_labeling_statistics(automated_df,manual_df): + """ + Function that will allow users to easily pass in two dataframes of manual labels and automated labels, + and a dataframe is returned with statistics examining the efficiency of the automated labelling system compared + to the human labels for multiple clips. + + Calls bird_local_scores on corresponding audio clips to generate the efficiency statistics for one specific clip which is then all put into one + dataframe of statistics for multiple audio clips. + + Args: + automated_df (Dataframe) - Dataframe of automated labels of multiple clips. + manual_df (Dataframe) - Dataframe of human labels of multiple clips. + + Returns: + Dataframe of statistics comparing automated labels and human labels for multiple clips. + """ # Getting a list of clips clips = automated_df["IN FILE"].to_list() # Removing duplicates @@ -590,8 +670,16 @@ def automated_labeling_statistics(automated_df,manual_df): statistics_df.reset_index(inplace = True, drop = True) return statistics_df -# Small function that takes in the statistics and outputs their global values def global_dataset_statistics(statistics_df): + """ + Function that takes in a dataframe of efficiency statistics for multiple clips and outputs their global values. + + Args: + statistics_df (Dataframe) - Dataframe of statistics value for multiple audio clips as returned by the function automated_labelling_statistics. + + Returns: + Dataframe of global statistics for the multiple audio clips' labelling. + """ tp_sum = statistics_df["TRUE POSITIVE"].sum() fp_sum = statistics_df["FALSE POSITIVE"].sum() fn_sum = statistics_df["FALSE NEGATIVE"].sum() @@ -610,6 +698,17 @@ def global_dataset_statistics(statistics_df): # TODO rework this function to implement some linear algebra, right now the nested for loop won't handle larger loads well # To make a global matrix, find the clip with the most amount of automated labels and set that to the number of columns def clip_IoU(automated_df,manual_df): + """ + Function that takes in the manual and automated labels for a clip and outputs human label-by-label IoU Scores. + + Args: + automated_df (Dataframe) - Dataframe of automated labels for an audio clip. + manual_df (Dataframe) - Dataframe of human labels for an audio clip. + + Returns: + Numpy Array of human label IoU scores. + """ + automated_df.reset_index(inplace = True, drop = True) manual_df.reset_index(inplace = True, drop = True) # Determining the number of rows in the output numpy array @@ -664,11 +763,21 @@ def clip_IoU(automated_df,manual_df): human_arr[human_arr == 1] = 0 return IoU_Matrix -# Function that takes in the IoU Matrix from the clip_IoU function and ouputs the number of true positives and false positives -# It also calculates the precision. + def matrix_IoU_Scores(IoU_Matrix,manual_df,threshold): - # This might not work in a situation where there is only one human label and multiple automated labels. - #IoU_Matrix_size = IoU_Matrix.shape[0] * IoU_Matrix.shape[1] + """ + Function that takes in the IoU Matrix from the clip_IoU function and ouputs the number of true positives and false positives, + as well as calculating the precision. + + Args: + IoU_Matrix (Numpy Array) - Matrix of human label IoU scores. + manual_df (Dataframe) - Dataframe of human labels for an audio clip. + threshold (float) - Threshold for determining true positives and false negatives. + + Returns: + Dataframe of clip statistics such as True Positive, False Negative, Precision, Recall, and F1 value. + """ + audio_dir = manual_df["FOLDER"][0] filename = manual_df["IN FILE"][0] @@ -708,8 +817,18 @@ def matrix_IoU_Scores(IoU_Matrix,manual_df,threshold): return pd.DataFrame.from_dict([entry]) -# Function that can help us determine whether or not a call was detected. def clip_catch(automated_df,manual_df): + """ + Function that determines the overlap between human and automated labels with respect to the number of samples in the human label. + + Args: + automated_df (Dataframe) - Dataframe of automated labels for an audio clip. + manual_df (Dataframe) - Dataframe of human labels for an audio clip. + + Returns: + Numpy Array of statistics regarding the amount of overlap between the manual and automated labels relative to the number of + samples. + """ # resetting the indices to make this function work automated_df.reset_index(inplace = True, drop = True) manual_df.reset_index(inplace = True, drop = True) @@ -755,10 +874,18 @@ def clip_catch(automated_df,manual_df): -# Function that takes in two Pandas dataframes that represent human labels and automated labels. -# It then runs the clip_IoU function across each clip and appends the best fit IoU score to each labels -# on the manual dataframe as its output. def dataset_IoU(automated_df,manual_df): + """ + Function that takes in two Pandas dataframes that represent human labels and automated labels. + It then runs the clip_IoU function across each clip and appends the best fit IoU score to each labels on the manual dataframe as its output. + + Args: + automated_df (Dataframe) - Dataframe of automated labels for multiple audio clips. + manual_df (Dataframe) - Dataframe of human labels for multiple audio clips. + + Returns: + Dataframe of manual labels with the best fit IoU score as a column. + """ # Getting a list of clips clips = automated_df["IN FILE"].to_list() # Removing duplicates @@ -786,6 +913,18 @@ def dataset_IoU(automated_df,manual_df): def dataset_IoU_Statistics(automated_df,manual_df,threshold = 0.5): + """ + Wrapper function that takes matrix_IoU_Scores across multiple clips. + Allows user to modify the threshold that determines whether or not a label is a true positive. + + Args: + automated_df (Dataframe) - Dataframe of automated labels for multiple audio clips. + manual_df (Dataframe) - Dataframe of human labels for multiple audio clips. + threshold (float) - Threshold for determining true positives. + + Returns: + Dataframe of IoU statistics for multiple audio clips. + """ # isolating the names of the clips that have been labelled into an array. clips = automated_df["IN FILE"].to_list() clips = list(dict.fromkeys(clips)) @@ -808,8 +947,18 @@ def dataset_IoU_Statistics(automated_df,manual_df,threshold = 0.5): IoU_Statistics = IoU_Statistics.append(clip_stats_df) IoU_Statistics.reset_index(inplace = True, drop = True) return IoU_Statistics -# Function that takes the output of dataset_IoU_Statistics and computes a global precision score. + def global_IoU_Statistics(statistics_df): + """ + Function that takes the output of dataset_IoU Statistics and outputs a global count of true positives and false positives, + as well as computing the precision across the dataset. + + Args: + statistics_df (Dataframe) - Dataframe of matrix IoU scores for multiple clips. + + Returns: + Dataframe of global IoU statistics. + """ # taking the sum of the number of true positives and false positives. tp_sum = statistics_df["TRUE POSITIVE"].sum() fn_sum = statistics_df["FALSE NEGATIVE"].sum() @@ -837,6 +986,16 @@ def global_IoU_Statistics(statistics_df): return pd.DataFrame.from_dict([entry]) def dataset_Catch(automated_df,manual_df): + """ + Function that determines the label-by-label "Catch" across multiple clips. + + Args: + automated_df (Dataframe) - Dataframe of automated labels for multiple audio clips. + manual_df (Dataframe) - Dataframe of human labels for multiple audio clips. + + Returns: + Dataframe of human labels with a column for the catch values of each label. + """ # Getting a list of clips clips = automated_df["IN FILE"].to_list() # Removing duplicates