From 26f429605d388c2905c03ff12ff6226f08a91f5a Mon Sep 17 00:00:00 2001 From: Jacob Date: Tue, 23 Mar 2021 11:39:37 -0700 Subject: [PATCH] Added in new "stack" isolation technique - Tested to make sure that it integrates into all of the other functions - Added True Negative values to IoU statistics, updated jupyter notebook accordingly. --- Microfaune_Local_Score_Package_Tutorial.ipynb | 1006 ++++++++++++++++- microfaune_local_score.py | 124 +- 2 files changed, 1093 insertions(+), 37 deletions(-) diff --git a/Microfaune_Local_Score_Package_Tutorial.ipynb b/Microfaune_Local_Score_Package_Tutorial.ipynb index b1d8360..022eeda 100644 --- a/Microfaune_Local_Score_Package_Tutorial.ipynb +++ b/Microfaune_Local_Score_Package_Tutorial.ipynb @@ -1091,7 +1091,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "WARNING:tensorflow:5 out of the last 8 calls to .predict_function at 0x7f798a34b5e0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" + "WARNING:tensorflow:5 out of the last 8 calls to .predict_function at 0x7ff58429d1f0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" ] }, { @@ -1134,7 +1134,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "WARNING:tensorflow:6 out of the last 9 calls to .predict_function at 0x7f798af54f70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" + "WARNING:tensorflow:6 out of the last 9 calls to .predict_function at 0x7ff5a2f40820> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" ] }, { @@ -1170,7 +1170,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "WARNING:tensorflow:7 out of the last 10 calls to .predict_function at 0x7f798904c9d0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" + "WARNING:tensorflow:7 out of the last 10 calls to .predict_function at 0x7ff542f0e670> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" ] }, { @@ -1212,7 +1212,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "WARNING:tensorflow:8 out of the last 11 calls to .predict_function at 0x7f7988699d30> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" + "WARNING:tensorflow:8 out of the last 11 calls to .predict_function at 0x7ff542599280> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" ] }, { @@ -1641,6 +1641,7 @@ " TRUE POSITIVE\n", " FALSE NEGATIVE\n", " FALSE POSITIVE\n", + " TRUE NEGATIVE\n", " PRECISION\n", " RECALL\n", " F1\n", @@ -1654,6 +1655,7 @@ " 1\n", " 4\n", " 4\n", + " 1\n", " 0.2\n", " 0.2\n", " 0.2\n", @@ -1666,8 +1668,11 @@ " FOLDER IN FILE \\\n", "0 /home/jacob/Acoustic-Species-Identification/pa... ScreamingPiha2.wav \n", "\n", - " TRUE POSITIVE FALSE NEGATIVE FALSE POSITIVE PRECISION RECALL F1 \n", - "0 1 4 4 0.2 0.2 0.2 " + " TRUE POSITIVE FALSE NEGATIVE FALSE POSITIVE TRUE NEGATIVE PRECISION \\\n", + "0 1 4 4 1 0.2 \n", + "\n", + " RECALL F1 \n", + "0 0.2 0.2 " ] }, "execution_count": 19, @@ -1736,6 +1741,7 @@ " TRUE POSITIVE\n", " FALSE NEGATIVE\n", " FALSE POSITIVE\n", + " TRUE NEGATIVE\n", " PRECISION\n", " RECALL\n", " F1\n", @@ -1749,6 +1755,7 @@ " 1\n", " 4\n", " 2\n", + " 1\n", " 0.3333\n", " 0.2\n", " 0.25\n", @@ -1760,6 +1767,7 @@ " 0\n", " 1\n", " 3\n", + " 0\n", " 0.0000\n", " 0.0\n", " 0.00\n", @@ -1771,6 +1779,7 @@ " 0\n", " 4\n", " 7\n", + " 0\n", " 0.0000\n", " 0.0\n", " 0.00\n", @@ -1782,6 +1791,7 @@ " 0\n", " 3\n", " 10\n", + " 0\n", " 0.0000\n", " 0.0\n", " 0.00\n", @@ -1793,6 +1803,7 @@ " 0\n", " 7\n", " 3\n", + " 0\n", " 0.0000\n", " 0.0\n", " 0.00\n", @@ -1804,6 +1815,7 @@ " 0\n", " 5\n", " 5\n", + " 0\n", " 0.0000\n", " 0.0\n", " 0.00\n", @@ -1821,13 +1833,21 @@ "4 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", "5 /home/jacob/Acoustic-Species-Identification/pa... ScreamingPiha2.wav \n", "\n", - " TRUE POSITIVE FALSE NEGATIVE FALSE POSITIVE PRECISION RECALL F1 \n", - "0 1 4 2 0.3333 0.2 0.25 \n", - "1 0 1 3 0.0000 0.0 0.00 \n", - "2 0 4 7 0.0000 0.0 0.00 \n", - "3 0 3 10 0.0000 0.0 0.00 \n", - "4 0 7 3 0.0000 0.0 0.00 \n", - "5 0 5 5 0.0000 0.0 0.00 " + " TRUE POSITIVE FALSE NEGATIVE FALSE POSITIVE TRUE NEGATIVE PRECISION \\\n", + "0 1 4 2 1 0.3333 \n", + "1 0 1 3 0 0.0000 \n", + "2 0 4 7 0 0.0000 \n", + "3 0 3 10 0 0.0000 \n", + "4 0 7 3 0 0.0000 \n", + "5 0 5 5 0 0.0000 \n", + "\n", + " RECALL F1 \n", + "0 0.2 0.25 \n", + "1 0.0 0.00 \n", + "2 0.0 0.00 \n", + "3 0.0 0.00 \n", + "4 0.0 0.00 \n", + "5 0.0 0.00 " ] }, "execution_count": 20, @@ -1876,6 +1896,7 @@ " TRUE POSITIVE\n", " FALSE NEGATIVE\n", " FALSE POSITIVE\n", + " TRUE NEGATIVE\n", " PRECISION\n", " RECALL\n", " F1\n", @@ -1887,6 +1908,7 @@ " 1\n", " 24\n", " 30\n", + " 1\n", " 0.0323\n", " 0.04\n", " 0.0357\n", @@ -1896,8 +1918,11 @@ "" ], "text/plain": [ - " TRUE POSITIVE FALSE NEGATIVE FALSE POSITIVE PRECISION RECALL F1\n", - "0 1 24 30 0.0323 0.04 0.0357" + " TRUE POSITIVE FALSE NEGATIVE FALSE POSITIVE TRUE NEGATIVE PRECISION \\\n", + "0 1 24 30 1 0.0323 \n", + "\n", + " RECALL F1 \n", + "0 0.04 0.0357 " ] }, "execution_count": 21, @@ -1909,6 +1934,957 @@ "global_stats_df = global_IoU_Statistics(stats_df)\n", "global_stats_df" ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "20190622_210000.WAV\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/jacob/Desktop/EngineersForExploration/Bioacoustics/Automated_Audio_Labelling_System_AID/microfaune_local_score.py:777: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " clip_manual_df[\"IoU\"] = automated_label_best_fits\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "20190623_222000.WAV\n", + "BlackFacedAntbird1.wav\n", + "HowlerMonkey1.WAV\n", + "20190624_152000.WAV\n", + "ScreamingPiha2.wav\n" + ] + } + ], + "source": [ + "manual_df_with_IoU = dataset_IoU(automated_df,manual_df)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
FOLDERIN FILECHANNELCLIP LENGTHOFFSETDURATIONMANUAL IDIoU
0/home/jacob/Acoustic-Species-Identification/pa...20190622_210000.WAV060.0000000.0009.70010.2848
1/home/jacob/Acoustic-Species-Identification/pa...20190622_210000.WAV060.00000011.50016.67510.4896
2/home/jacob/Acoustic-Species-Identification/pa...20190622_210000.WAV060.00000029.50015.00010.4345
3/home/jacob/Acoustic-Species-Identification/pa...20190622_210000.WAV060.00000048.5251.00010.0000
4/home/jacob/Acoustic-Species-Identification/pa...20190622_210000.WAV060.00000050.5008.00010.8444
5/home/jacob/Acoustic-Species-Identification/pa...20190623_222000.WAV060.00000014.50012.50010.6778
6/home/jacob/Acoustic-Species-Identification/pa...BlackFacedAntbird1.wav031.2000000.0004.10010.3344
7/home/jacob/Acoustic-Species-Identification/pa...BlackFacedAntbird1.wav031.2000005.0001.50010.3447
8/home/jacob/Acoustic-Species-Identification/pa...BlackFacedAntbird1.wav031.2000009.8005.20010.2836
9/home/jacob/Acoustic-Species-Identification/pa...BlackFacedAntbird1.wav031.20000021.5005.50010.1914
10/home/jacob/Acoustic-Species-Identification/pa...HowlerMonkey1.WAV060.0000000.00010.90010.4410
11/home/jacob/Acoustic-Species-Identification/pa...HowlerMonkey1.WAV060.00000011.50013.00010.1556
12/home/jacob/Acoustic-Species-Identification/pa...HowlerMonkey1.WAV060.00000025.00034.90010.1637
13/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV060.0000001.2000.45010.0000
14/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV060.0000004.0000.30010.0000
15/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV060.0000007.4000.60010.0512
16/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV060.00000010.5000.80010.1566
17/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV060.00000020.8000.50010.0000
18/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV060.00000028.0000.40010.0000
19/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV060.00000034.0001.00010.0000
20/home/jacob/Acoustic-Species-Identification/pa...ScreamingPiha2.wav033.9330610.0005.37310.4394
21/home/jacob/Acoustic-Species-Identification/pa...ScreamingPiha2.wav033.93306110.5905.58510.4412
22/home/jacob/Acoustic-Species-Identification/pa...ScreamingPiha2.wav033.93306122.0004.00010.7331
23/home/jacob/Acoustic-Species-Identification/pa...ScreamingPiha2.wav033.93306127.2000.90010.2690
24/home/jacob/Acoustic-Species-Identification/pa...ScreamingPiha2.wav033.93306129.0004.50010.4809
\n", + "
" + ], + "text/plain": [ + " FOLDER IN FILE \\\n", + "0 /home/jacob/Acoustic-Species-Identification/pa... 20190622_210000.WAV \n", + "1 /home/jacob/Acoustic-Species-Identification/pa... 20190622_210000.WAV \n", + "2 /home/jacob/Acoustic-Species-Identification/pa... 20190622_210000.WAV \n", + "3 /home/jacob/Acoustic-Species-Identification/pa... 20190622_210000.WAV \n", + "4 /home/jacob/Acoustic-Species-Identification/pa... 20190622_210000.WAV \n", + "5 /home/jacob/Acoustic-Species-Identification/pa... 20190623_222000.WAV \n", + "6 /home/jacob/Acoustic-Species-Identification/pa... BlackFacedAntbird1.wav \n", + "7 /home/jacob/Acoustic-Species-Identification/pa... BlackFacedAntbird1.wav \n", + "8 /home/jacob/Acoustic-Species-Identification/pa... BlackFacedAntbird1.wav \n", + "9 /home/jacob/Acoustic-Species-Identification/pa... BlackFacedAntbird1.wav \n", + "10 /home/jacob/Acoustic-Species-Identification/pa... HowlerMonkey1.WAV \n", + "11 /home/jacob/Acoustic-Species-Identification/pa... HowlerMonkey1.WAV \n", + "12 /home/jacob/Acoustic-Species-Identification/pa... HowlerMonkey1.WAV \n", + "13 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", + "14 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", + "15 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", + "16 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", + "17 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", + "18 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", + "19 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", + "20 /home/jacob/Acoustic-Species-Identification/pa... ScreamingPiha2.wav \n", + "21 /home/jacob/Acoustic-Species-Identification/pa... ScreamingPiha2.wav \n", + "22 /home/jacob/Acoustic-Species-Identification/pa... ScreamingPiha2.wav \n", + "23 /home/jacob/Acoustic-Species-Identification/pa... ScreamingPiha2.wav \n", + "24 /home/jacob/Acoustic-Species-Identification/pa... ScreamingPiha2.wav \n", + "\n", + " CHANNEL CLIP LENGTH OFFSET DURATION MANUAL ID IoU \n", + "0 0 60.000000 0.000 9.700 1 0.2848 \n", + "1 0 60.000000 11.500 16.675 1 0.4896 \n", + "2 0 60.000000 29.500 15.000 1 0.4345 \n", + "3 0 60.000000 48.525 1.000 1 0.0000 \n", + "4 0 60.000000 50.500 8.000 1 0.8444 \n", + "5 0 60.000000 14.500 12.500 1 0.6778 \n", + "6 0 31.200000 0.000 4.100 1 0.3344 \n", + "7 0 31.200000 5.000 1.500 1 0.3447 \n", + "8 0 31.200000 9.800 5.200 1 0.2836 \n", + "9 0 31.200000 21.500 5.500 1 0.1914 \n", + "10 0 60.000000 0.000 10.900 1 0.4410 \n", + "11 0 60.000000 11.500 13.000 1 0.1556 \n", + "12 0 60.000000 25.000 34.900 1 0.1637 \n", + "13 0 60.000000 1.200 0.450 1 0.0000 \n", + "14 0 60.000000 4.000 0.300 1 0.0000 \n", + "15 0 60.000000 7.400 0.600 1 0.0512 \n", + "16 0 60.000000 10.500 0.800 1 0.1566 \n", + "17 0 60.000000 20.800 0.500 1 0.0000 \n", + "18 0 60.000000 28.000 0.400 1 0.0000 \n", + "19 0 60.000000 34.000 1.000 1 0.0000 \n", + "20 0 33.933061 0.000 5.373 1 0.4394 \n", + "21 0 33.933061 10.590 5.585 1 0.4412 \n", + "22 0 33.933061 22.000 4.000 1 0.7331 \n", + "23 0 33.933061 27.200 0.900 1 0.2690 \n", + "24 0 33.933061 29.000 4.500 1 0.4809 " + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "manual_df_with_IoU" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Function that determines the overlap between human and automated labels with respect to the number of samples in the human label. Referred to as \"Catch\"" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[1. 0.7344 1. 1. 0.7588]\n" + ] + } + ], + "source": [ + "Catch_Array = clip_catch(automated_piha_df,manual_piha_df)\n", + "print(Catch_Array)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Function that determines the label-by-label \"Catch\" across multiple clips" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "20190622_210000.WAV\n", + "20190623_222000.WAV\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/jacob/Desktop/EngineersForExploration/Bioacoustics/Automated_Audio_Labelling_System_AID/microfaune_local_score.py:855: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " clip_manual_df[\"Catch\"] = Catch_Array\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "BlackFacedAntbird1.wav\n", + "HowlerMonkey1.WAV\n", + "20190624_152000.WAV\n", + "ScreamingPiha2.wav\n" + ] + } + ], + "source": [ + "manual_df_with_catch = dataset_Catch(automated_df,manual_df)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
FOLDERIN FILECHANNELCLIP LENGTHOFFSETDURATIONMANUAL IDCatch
0/home/jacob/Acoustic-Species-Identification/pa...20190622_210000.WAV060.0000000.0009.70011.0000
1/home/jacob/Acoustic-Species-Identification/pa...20190622_210000.WAV060.00000011.50016.67511.0000
2/home/jacob/Acoustic-Species-Identification/pa...20190622_210000.WAV060.00000029.50015.00010.7885
3/home/jacob/Acoustic-Species-Identification/pa...20190622_210000.WAV060.00000048.5251.00010.0000
4/home/jacob/Acoustic-Species-Identification/pa...20190622_210000.WAV060.00000050.5008.00010.8444
5/home/jacob/Acoustic-Species-Identification/pa...20190623_222000.WAV060.00000014.50012.50010.6778
6/home/jacob/Acoustic-Species-Identification/pa...BlackFacedAntbird1.wav031.2000000.0004.10010.6448
7/home/jacob/Acoustic-Species-Identification/pa...BlackFacedAntbird1.wav031.2000005.0001.50010.8951
8/home/jacob/Acoustic-Species-Identification/pa...BlackFacedAntbird1.wav031.2000009.8005.20010.4523
9/home/jacob/Acoustic-Species-Identification/pa...BlackFacedAntbird1.wav031.20000021.5005.50010.4737
10/home/jacob/Acoustic-Species-Identification/pa...HowlerMonkey1.WAV060.0000000.00010.90010.8562
11/home/jacob/Acoustic-Species-Identification/pa...HowlerMonkey1.WAV060.00000011.50013.00010.4616
12/home/jacob/Acoustic-Species-Identification/pa...HowlerMonkey1.WAV060.00000025.00034.90010.6903
13/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV060.0000001.2000.45010.0000
14/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV060.0000004.0000.30010.0000
15/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV060.0000007.4000.60010.4630
16/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV060.00000010.5000.80011.0000
17/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV060.00000020.8000.50010.0000
18/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV060.00000028.0000.40010.0000
19/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV060.00000034.0001.00010.0000
20/home/jacob/Acoustic-Species-Identification/pa...ScreamingPiha2.wav033.9330610.0005.37311.0000
21/home/jacob/Acoustic-Species-Identification/pa...ScreamingPiha2.wav033.93306110.5905.58510.7344
22/home/jacob/Acoustic-Species-Identification/pa...ScreamingPiha2.wav033.93306122.0004.00011.0000
23/home/jacob/Acoustic-Species-Identification/pa...ScreamingPiha2.wav033.93306127.2000.90011.0000
24/home/jacob/Acoustic-Species-Identification/pa...ScreamingPiha2.wav033.93306129.0004.50010.7588
\n", + "
" + ], + "text/plain": [ + " FOLDER IN FILE \\\n", + "0 /home/jacob/Acoustic-Species-Identification/pa... 20190622_210000.WAV \n", + "1 /home/jacob/Acoustic-Species-Identification/pa... 20190622_210000.WAV \n", + "2 /home/jacob/Acoustic-Species-Identification/pa... 20190622_210000.WAV \n", + "3 /home/jacob/Acoustic-Species-Identification/pa... 20190622_210000.WAV \n", + "4 /home/jacob/Acoustic-Species-Identification/pa... 20190622_210000.WAV \n", + "5 /home/jacob/Acoustic-Species-Identification/pa... 20190623_222000.WAV \n", + "6 /home/jacob/Acoustic-Species-Identification/pa... BlackFacedAntbird1.wav \n", + "7 /home/jacob/Acoustic-Species-Identification/pa... BlackFacedAntbird1.wav \n", + "8 /home/jacob/Acoustic-Species-Identification/pa... BlackFacedAntbird1.wav \n", + "9 /home/jacob/Acoustic-Species-Identification/pa... BlackFacedAntbird1.wav \n", + "10 /home/jacob/Acoustic-Species-Identification/pa... HowlerMonkey1.WAV \n", + "11 /home/jacob/Acoustic-Species-Identification/pa... HowlerMonkey1.WAV \n", + "12 /home/jacob/Acoustic-Species-Identification/pa... HowlerMonkey1.WAV \n", + "13 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", + "14 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", + "15 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", + "16 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", + "17 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", + "18 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", + "19 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", + "20 /home/jacob/Acoustic-Species-Identification/pa... ScreamingPiha2.wav \n", + "21 /home/jacob/Acoustic-Species-Identification/pa... ScreamingPiha2.wav \n", + "22 /home/jacob/Acoustic-Species-Identification/pa... ScreamingPiha2.wav \n", + "23 /home/jacob/Acoustic-Species-Identification/pa... ScreamingPiha2.wav \n", + "24 /home/jacob/Acoustic-Species-Identification/pa... ScreamingPiha2.wav \n", + "\n", + " CHANNEL CLIP LENGTH OFFSET DURATION MANUAL ID Catch \n", + "0 0 60.000000 0.000 9.700 1 1.0000 \n", + "1 0 60.000000 11.500 16.675 1 1.0000 \n", + "2 0 60.000000 29.500 15.000 1 0.7885 \n", + "3 0 60.000000 48.525 1.000 1 0.0000 \n", + "4 0 60.000000 50.500 8.000 1 0.8444 \n", + "5 0 60.000000 14.500 12.500 1 0.6778 \n", + "6 0 31.200000 0.000 4.100 1 0.6448 \n", + "7 0 31.200000 5.000 1.500 1 0.8951 \n", + "8 0 31.200000 9.800 5.200 1 0.4523 \n", + "9 0 31.200000 21.500 5.500 1 0.4737 \n", + "10 0 60.000000 0.000 10.900 1 0.8562 \n", + "11 0 60.000000 11.500 13.000 1 0.4616 \n", + "12 0 60.000000 25.000 34.900 1 0.6903 \n", + "13 0 60.000000 1.200 0.450 1 0.0000 \n", + "14 0 60.000000 4.000 0.300 1 0.0000 \n", + "15 0 60.000000 7.400 0.600 1 0.4630 \n", + "16 0 60.000000 10.500 0.800 1 1.0000 \n", + "17 0 60.000000 20.800 0.500 1 0.0000 \n", + "18 0 60.000000 28.000 0.400 1 0.0000 \n", + "19 0 60.000000 34.000 1.000 1 0.0000 \n", + "20 0 33.933061 0.000 5.373 1 1.0000 \n", + "21 0 33.933061 10.590 5.585 1 0.7344 \n", + "22 0 33.933061 22.000 4.000 1 1.0000 \n", + "23 0 33.933061 27.200 0.900 1 1.0000 \n", + "24 0 33.933061 29.000 4.500 1 0.7588 " + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "manual_df_with_catch" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'calc_local_scores_simpler' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mnew_automated_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcalc_local_scores_simpler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mNameError\u001b[0m: name 'calc_local_scores_simpler' is not defined" + ] + } + ], + "source": [ + "new_automated_df = calc_local_scores_simpler(path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "new_automated_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "local_score_visualization2(\"./TEST/ScreamingPiha2.wav\",automated_df = True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "stats_df2 = dataset_IoU_Statistics(new_automated_df,manual_df,threshold = 0.5)\n", + "stats_df2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "global_stats_df = global_IoU_Statistics(stats_df2)\n", + "global_stats_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "annotation_duration_statistics(new_automated_df)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/microfaune_local_score.py b/microfaune_local_score.py index c35d474..0de553b 100644 --- a/microfaune_local_score.py +++ b/microfaune_local_score.py @@ -17,14 +17,8 @@ import pandas as pd import math -# Gabriel's original moment-to-moment classification tool. Reworked to output -# a Pandas DataFrame. -# TODO rework isolate in a way that allows a user to input a dictionary that where they can modulate different -# parameters involved in Gabriel's algorithm. We can set the default of this dict to be what he originally chose for now. -# Some ideas for how to change the parameters are to allow for different modification of the threshold. We would want to be able -# to modify the bird presence threshold to be a pure value. This will allow us to build ROC curves. Another would be to allow for a -# selection of how many standard deviations away from the mean. Another would be, instead of a median, allow standard deviation and mean as -# alternatives. Another option would be to allow for curve smoothing on the local score array that is being passed in. This could come in + +# Another option would be to allow for curve smoothing on the local score array that is being passed in. This could come in # the form of a high order polynomial fit or possibly testing out my curve smoothing algorithm that uses a bell-curved distribution to # loop around and average each sample with its surrounding samples over many iterations. We could also play around with filtering. @@ -36,10 +30,13 @@ def isolate(local_scores, SIGNAL, SAMPLE_RATE, audio_dir, filename,isolation_par # deciding which isolation technique to deploy for a given clip if isolation_parameters["technique"] == "simple": - isolation_df = simple_isolate(local_scores, SIGNAL, SAMPLE_RATE, audio_dir, filename, isolation_parameters,manual_id = "bird") + isolation_df = simple_isolate(local_scores, SIGNAL, SAMPLE_RATE, audio_dir, filename, isolation_parameters, manual_id = "bird") elif isolation_parameters["technique"] == "steinberg": - isolation_df = steinberg_isolate(local_scores, SIGNAL, SAMPLE_RATE, audio_dir, filename,isolation_parameters,manual_id = "bird") -# elif isolation_parameters["technique"] == "stack" + isolation_df = steinberg_isolate(local_scores, SIGNAL, SAMPLE_RATE, audio_dir, filename,isolation_parameters, manual_id = "bird") + elif isolation_parameters["technique"] == "stack": + isolation_df = stack_isolate(local_scores, SIGNAL, SAMPLE_RATE, audio_dir, filename, isolation_parameters, manual_id = "bird") + # stack_isolate(local_scores, SIGNAL, SAMPLE_RATE, audio_dir, filename, isolation_parameters, manual_id = "bird"): + return isolation_df @@ -128,6 +125,7 @@ def steinberg_isolate(local_scores, SIGNAL, SAMPLE_RATE, audio_dir, filename,iso return entry def simple_isolate(local_scores, SIGNAL, SAMPLE_RATE, audio_dir, filename, isolation_parameters, manual_id = "bird"): + #local_scores2 = local_scores #threshold = 2*np.median(local_scores) if isolation_parameters["threshold_type"] == "median": @@ -160,25 +158,22 @@ def simple_isolate(local_scores, SIGNAL, SAMPLE_RATE, audio_dir, filename, isola # local_score * samples_per_score / sample_rate time_per_score = samples_per_score / SAMPLE_RATE - # setting scores above the threshold equal to one - #local_scores2[local_scores2 >= threshold] = 1 - - consecutive_samples = 0 + annotation_start = 0 call_start = 0 call_stop = 0 # looping through all of the local scores for ndx in range(len(local_scores)): current_score = local_scores[ndx] # Start of a new sequence. - if current_score >= thresh and consecutive_samples == 0: + if current_score >= thresh and annotation_start == 0: # signal a start of a new sequence. - consecutive_samples = 1 + annotation_start = 1 call_start = float(ndx*time_per_score) #print("Call Start",call_start) # End of a sequence - elif current_score < thresh and consecutive_samples == 1: + elif current_score < thresh and annotation_start == 1: # signal the end of a sequence - consecutive_samples = 0 + annotation_start = 0 # call_end = float(ndx*time_per_score) #print("Call End",call_end) @@ -190,11 +185,90 @@ def simple_isolate(local_scores, SIGNAL, SAMPLE_RATE, audio_dir, filename, isola else: continue return pd.DataFrame.from_dict(entry) - # implement this function after -# def stack_isolate(local_scores, SIGNAL, SAMPLE_RATE, audio_dir, filename, threshold_type = "median", threshold_const = 2.0): +def stack_isolate(local_scores, SIGNAL, SAMPLE_RATE, audio_dir, filename, isolation_parameters, manual_id = "bird"): + + # configuring the threshold based on isolation parameters + if isolation_parameters["threshold_type"] == "median": + thresh = np.median(local_scores) * isolation_parameters["threshold_const"] + elif isolation_parameters["threshold_type"] == "mean" or isolation_parameters["threshold_type"] == "average": + thresh = np.mean(local_scores) * isolation_parameters["threshold_const"] + elif isolation_parameters["threshold_type"] == "standard deviation": + thresh = np.std(local_scores) * isolation_parameters["threshold_const"] + elif isolation_parameters["threshold_type"] == "pure" and (isolation_parameters["threshold_const"] < 0 or isolation_parameters["threshold_const"] > 1): + print("A pure threshold must be between [0,1], exiting function") + return + elif isolation_parameters["threshold_type"] == "pure": + thresh = isolation_parameters["threshold_const"] + + # calculate original duration + old_duration = len(SIGNAL) / SAMPLE_RATE + + # initializing a dictionary that will be used to construct the output pandas dataframe. + entry = {'FOLDER' : audio_dir, + 'IN FILE' : filename, + 'CHANNEL' : 0, + 'CLIP LENGTH': old_duration, + 'SAMPLE RATE': SAMPLE_RATE, + 'OFFSET' : [], + 'DURATION' : [], + 'MANUAL ID' : []} + + # how many samples one score represents + # Scores meaning local scores + samples_per_score = len(SIGNAL) // len(local_scores) + # local_score * samples_per_score / sample_rate + # constant that will be used to convert from local score indices to annotation start/stop values. + time_per_score = samples_per_score / SAMPLE_RATE + + # initializing variables used in master loop + stack_counter = 0 + annotation_start = 0 + call_start = 0 + call_stop = 0 + # looping through every local score array value + for ndx in range(len(local_scores)): + # the case for the end of the local score array and the stack isn't empty. + if ndx == (len(local_scores) - 1) and stack_counter > 0: + call_end = float(ndx*time_per_score) + entry['OFFSET'].append(call_start) + entry['DURATION'].append(call_end - call_start) + entry['MANUAL ID'].append(manual_id) + # pushing onto the stack whenever a sample is above the threshold + if local_scores[ndx] >= thresh: + # in case this is the start of a new annotation + if stack_counter == 0: + call_start = float(ndx*time_per_score) + annotation_start = 1 + # increasing this stack counter will be referred to as "pushing" + stack_counter = stack_counter + 1 + + # when a score is below the treshold + else: + # the case where it is the end of an annotation + if stack_counter == 0 and annotation_start == 1: + # marking the end of a clip + call_end = float(ndx*time_per_score) + + # adding annotation to dictionary containing all annotations + entry['OFFSET'].append(call_start) + entry['DURATION'].append(call_end - call_start) + entry['MANUAL ID'].append(manual_id) + + # resetting for the next annotation + call_start = 0 + call_end = 0 + annotation_start = 0 + # the case where the stack is empty and a new annotation hasn't started, you just want to increment the index + elif stack_counter == 0 and annotation_start == 0: + continue + # the case where we are below the threshold and the stack isn't empty. Pop from the stack, which in this case means just subtracting from the counter. + else: + stack_counter = stack_counter - 1 + # returning pandas dataframe from dictionary constructed with all of the annotations + return pd.DataFrame.from_dict(entry) ## Function that applies the moment to moment labeling system to a directory full of wav files. def generate_automated_labels(bird_dir, isolation_parameters, weight_path=None, Normalized_Sample_Rate = 44100): @@ -243,6 +317,7 @@ def generate_automated_labels(bird_dir, isolation_parameters, weight_path=None, # get duration of clip duration = len(SIGNAL) / SAMPLE_RATE + try: # Running moment to moment algorithm and appending to a master dataframe. new_entry = isolate(local_scores[0], SIGNAL, SAMPLE_RATE, bird_dir, audio_file, isolation_parameters, manual_id = "bird") @@ -592,7 +667,8 @@ def clip_IoU(automated_df,manual_df): # Function that takes in the IoU Matrix from the clip_IoU function and ouputs the number of true positives and false positives # It also calculates the precision. def matrix_IoU_Scores(IoU_Matrix,manual_df,threshold): - + # This might not work in a situation where there is only one human label and multiple automated labels. + #IoU_Matrix_size = IoU_Matrix.shape[0] * IoU_Matrix.shape[1] audio_dir = manual_df["FOLDER"][0] filename = manual_df["IN FILE"][0] @@ -607,6 +683,7 @@ def matrix_IoU_Scores(IoU_Matrix,manual_df,threshold): # Calculating the false positives max_val_per_column = np.max(IoU_Matrix,axis=0) fp_count = max_val_per_column[max_val_per_column < threshold].shape[0] + tn_count = max_val_per_column[max_val_per_column >= threshold].shape[0] # Calculating the necessary statistics try: @@ -624,6 +701,7 @@ def matrix_IoU_Scores(IoU_Matrix,manual_df,threshold): 'TRUE POSITIVE' : tp_count, 'FALSE NEGATIVE' : fn_count, 'FALSE POSITIVE': fp_count, + 'TRUE NEGATIVE' : tn_count, 'PRECISION' : precision, 'RECALL' : recall, 'F1' : f1} @@ -736,6 +814,7 @@ def global_IoU_Statistics(statistics_df): tp_sum = statistics_df["TRUE POSITIVE"].sum() fn_sum = statistics_df["FALSE NEGATIVE"].sum() fp_sum = statistics_df["FALSE POSITIVE"].sum() + tn_sum = statistics_df["TRUE NEGATIVE"].sum() # calculating the precision, recall, and f1 try: precision = tp_sum/(tp_sum+fp_sum) @@ -750,6 +829,7 @@ def global_IoU_Statistics(statistics_df): entry = {'TRUE POSITIVE' : tp_sum, 'FALSE NEGATIVE' : fn_sum, 'FALSE POSITIVE' : fp_sum, + 'TRUE NEGATIVE' : tn_sum, 'PRECISION' : round(precision,4), 'RECALL' : round(recall,4), 'F1' : round(f1,4)}