From cda7e6f8622f1ddc2ff1d56e57f22680e6adf421 Mon Sep 17 00:00:00 2001 From: Jacob Date: Wed, 21 Apr 2021 22:53:48 -0700 Subject: [PATCH] Added Kaleidoscope converting function - Function strips away columns in the manual or automated dataframes that we need for our package that would otherewise be incompatible with Kaleidoscope. --- IsoAutio/isolation.py | 9 + IsoAutio_Tutorial.ipynb | 386 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 370 insertions(+), 25 deletions(-) diff --git a/IsoAutio/isolation.py b/IsoAutio/isolation.py index 74374c6..f7fbc4f 100644 --- a/IsoAutio/isolation.py +++ b/IsoAutio/isolation.py @@ -421,3 +421,12 @@ def generate_automated_labels(bird_dir, isolation_parameters, weight_path=None, # Quick fix to indexing annotations.reset_index(inplace = True, drop = True) return annotations + + +def kaleidoscope_conversion(df): + kaleidoscope_df = [df["FOLDER"], df["IN FILE"], df["CHANNEL"], df["OFFSET"], df["DURATION"], df["MANUAL ID"]] + headers = ["FOLDER", "IN FILE", "CHANNEL", "OFFSET", "DURATION", "MANUAL ID"] + + + kaleidoscope_df = pd.concat(kaleidoscope_df, axis=1, keys=headers) + return kaleidoscope_df diff --git a/IsoAutio_Tutorial.ipynb b/IsoAutio_Tutorial.ipynb index 6b7443d..3488154 100644 --- a/IsoAutio_Tutorial.ipynb +++ b/IsoAutio_Tutorial.ipynb @@ -1037,6 +1037,342 @@ "annotation_duration_statistics(manual_df)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Helper function to convert to kaleidoscope-compatible format " + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
FOLDERIN FILECHANNELOFFSETDURATIONMANUAL ID
0/home/jacob/Acoustic-Species-Identification/pa...ScreamingPiha2.wav00.0005.3731
1/home/jacob/Acoustic-Species-Identification/pa...ScreamingPiha2.wav010.5905.5851
2/home/jacob/Acoustic-Species-Identification/pa...ScreamingPiha2.wav022.0004.0001
3/home/jacob/Acoustic-Species-Identification/pa...ScreamingPiha2.wav027.2000.9001
4/home/jacob/Acoustic-Species-Identification/pa...ScreamingPiha2.wav029.0004.5001
5/home/jacob/Acoustic-Species-Identification/pa...20190622_210000.WAV00.0009.7001
6/home/jacob/Acoustic-Species-Identification/pa...20190622_210000.WAV011.50016.6751
7/home/jacob/Acoustic-Species-Identification/pa...20190622_210000.WAV029.50015.0001
8/home/jacob/Acoustic-Species-Identification/pa...20190622_210000.WAV048.5251.0001
9/home/jacob/Acoustic-Species-Identification/pa...20190622_210000.WAV050.5008.0001
10/home/jacob/Acoustic-Species-Identification/pa...20190623_222000.WAV014.50012.5001
11/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV01.2000.4501
12/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV04.0000.3001
13/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV07.4000.6001
14/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV010.5000.8001
15/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV020.8000.5001
16/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV028.0000.4001
17/home/jacob/Acoustic-Species-Identification/pa...20190624_152000.WAV034.0001.0001
18/home/jacob/Acoustic-Species-Identification/pa...BlackFacedAntbird1.wav00.0004.1001
19/home/jacob/Acoustic-Species-Identification/pa...BlackFacedAntbird1.wav05.0001.5001
20/home/jacob/Acoustic-Species-Identification/pa...BlackFacedAntbird1.wav09.8005.2001
21/home/jacob/Acoustic-Species-Identification/pa...BlackFacedAntbird1.wav021.5005.5001
22/home/jacob/Acoustic-Species-Identification/pa...HowlerMonkey1.WAV00.00010.9001
23/home/jacob/Acoustic-Species-Identification/pa...HowlerMonkey1.WAV011.50013.0001
24/home/jacob/Acoustic-Species-Identification/pa...HowlerMonkey1.WAV025.00034.9001
\n", + "
" + ], + "text/plain": [ + " FOLDER IN FILE \\\n", + "0 /home/jacob/Acoustic-Species-Identification/pa... ScreamingPiha2.wav \n", + "1 /home/jacob/Acoustic-Species-Identification/pa... ScreamingPiha2.wav \n", + "2 /home/jacob/Acoustic-Species-Identification/pa... ScreamingPiha2.wav \n", + "3 /home/jacob/Acoustic-Species-Identification/pa... ScreamingPiha2.wav \n", + "4 /home/jacob/Acoustic-Species-Identification/pa... ScreamingPiha2.wav \n", + "5 /home/jacob/Acoustic-Species-Identification/pa... 20190622_210000.WAV \n", + "6 /home/jacob/Acoustic-Species-Identification/pa... 20190622_210000.WAV \n", + "7 /home/jacob/Acoustic-Species-Identification/pa... 20190622_210000.WAV \n", + "8 /home/jacob/Acoustic-Species-Identification/pa... 20190622_210000.WAV \n", + "9 /home/jacob/Acoustic-Species-Identification/pa... 20190622_210000.WAV \n", + "10 /home/jacob/Acoustic-Species-Identification/pa... 20190623_222000.WAV \n", + "11 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", + "12 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", + "13 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", + "14 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", + "15 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", + "16 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", + "17 /home/jacob/Acoustic-Species-Identification/pa... 20190624_152000.WAV \n", + "18 /home/jacob/Acoustic-Species-Identification/pa... BlackFacedAntbird1.wav \n", + "19 /home/jacob/Acoustic-Species-Identification/pa... BlackFacedAntbird1.wav \n", + "20 /home/jacob/Acoustic-Species-Identification/pa... BlackFacedAntbird1.wav \n", + "21 /home/jacob/Acoustic-Species-Identification/pa... BlackFacedAntbird1.wav \n", + "22 /home/jacob/Acoustic-Species-Identification/pa... HowlerMonkey1.WAV \n", + "23 /home/jacob/Acoustic-Species-Identification/pa... HowlerMonkey1.WAV \n", + "24 /home/jacob/Acoustic-Species-Identification/pa... HowlerMonkey1.WAV \n", + "\n", + " CHANNEL OFFSET DURATION MANUAL ID \n", + "0 0 0.000 5.373 1 \n", + "1 0 10.590 5.585 1 \n", + "2 0 22.000 4.000 1 \n", + "3 0 27.200 0.900 1 \n", + "4 0 29.000 4.500 1 \n", + "5 0 0.000 9.700 1 \n", + "6 0 11.500 16.675 1 \n", + "7 0 29.500 15.000 1 \n", + "8 0 48.525 1.000 1 \n", + "9 0 50.500 8.000 1 \n", + "10 0 14.500 12.500 1 \n", + "11 0 1.200 0.450 1 \n", + "12 0 4.000 0.300 1 \n", + "13 0 7.400 0.600 1 \n", + "14 0 10.500 0.800 1 \n", + "15 0 20.800 0.500 1 \n", + "16 0 28.000 0.400 1 \n", + "17 0 34.000 1.000 1 \n", + "18 0 0.000 4.100 1 \n", + "19 0 5.000 1.500 1 \n", + "20 0 9.800 5.200 1 \n", + "21 0 21.500 5.500 1 \n", + "22 0 0.000 10.900 1 \n", + "23 0 11.500 13.000 1 \n", + "24 0 25.000 34.900 1 " + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kaleidoscope_conversion(manual_df)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -1053,7 +1389,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": { "scrolled": false }, @@ -1092,14 +1428,14 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "WARNING:tensorflow:5 out of the last 8 calls to .predict_function at 0x7f460d2ab790> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" + "WARNING:tensorflow:5 out of the last 8 calls to .predict_function at 0x7f12d69dfc10> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" ] }, { @@ -1135,14 +1471,14 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "WARNING:tensorflow:6 out of the last 9 calls to .predict_function at 0x7f4623f6ce50> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" + "WARNING:tensorflow:6 out of the last 9 calls to .predict_function at 0x7f12d75b4310> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" ] }, { @@ -1178,14 +1514,14 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "WARNING:tensorflow:7 out of the last 10 calls to .predict_function at 0x7f45c7ecc4c0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" + "WARNING:tensorflow:7 out of the last 10 calls to .predict_function at 0x7f12d5514940> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" ] }, { @@ -1214,14 +1550,14 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "WARNING:tensorflow:8 out of the last 11 calls to .predict_function at 0x7f460c118700> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" + "WARNING:tensorflow:8 out of the last 11 calls to .predict_function at 0x7f12d604b430> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" ] }, { @@ -1254,7 +1590,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "metadata": { "scrolled": true }, @@ -1263,7 +1599,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "WARNING:tensorflow:9 out of the last 12 calls to .predict_function at 0x7f45c7ebb1f0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" + "WARNING:tensorflow:9 out of the last 12 calls to .predict_function at 0x7f12d75565e0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" ] }, { @@ -1292,7 +1628,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -1316,7 +1652,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -1332,7 +1668,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -1484,7 +1820,7 @@ "5 0.628725 " ] }, - "execution_count": 17, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } @@ -1503,7 +1839,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -1550,7 +1886,7 @@ "0 0.727076 0.743082 0.734992 0.581018" ] }, - "execution_count": 18, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -1568,7 +1904,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -1597,7 +1933,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -1655,7 +1991,7 @@ "0 1 4 4 0.2 0.2 0.2 " ] }, - "execution_count": 20, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -1673,7 +2009,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 22, "metadata": { "scrolled": true }, @@ -1815,7 +2151,7 @@ "5 0 5 5 0.0000 0.0 0.00 " ] }, - "execution_count": 21, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } @@ -1834,7 +2170,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 23, "metadata": {}, "outputs": [ { @@ -1885,7 +2221,7 @@ "0 1 24 30 0.0323 0.04 0.0357" ] }, - "execution_count": 22, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } @@ -1897,7 +2233,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 24, "metadata": { "scrolled": true },