src/2-analyze-source-collection-and-target.ipynb

{"cells":[{"cell_type":"markdown","metadata":{"id":"3CVvi4YnddIp"},"source":["# 2) Analyze source collection and target file\n","\n","This notebook includes the code to analyze the collection of sounds compiled in the previous notebook and that will be later used as the source collection in our audio mosaicing code. The notebook also contains the code to analyze the target audio file that will be later reconstructed using sound chunks from the source collection.\n","\n","The audio analysis carried out in this notebook uses the Pythonn bindings of the Essentia library which was introduced in the first session of AMPLAB. Please make sure you checked the [Essentia Python tutorial](https://essentia.upf.edu/documentation/essentia_python_tutorial.html) to get familiarized with using Essentia in Python. Also useful is to always have a browser tab opened with Essentia's [Algorithms Reference](https://essentia.upf.edu/documentation/algorithms_reference.html) documentation page."]},{"cell_type":"code","execution_count":1,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":10796,"status":"ok","timestamp":1648945798158,"user":{"displayName":"PATRICIO OVALLE","userId":"13329772063495446668"},"user_tz":420},"id":"IBsJAzcI7aPw","outputId":"36fa4232-d188-4cb5-e583-ae3d2cc2f7fc"},"outputs":[{"output_type":"stream","name":"stdout","text":["Requirement already satisfied: essentia in /usr/local/lib/python3.7/dist-packages (2.1b6.dev778)\n","Requirement already satisfied: numpy>=1.8.2 in /usr/local/lib/python3.7/dist-packages (from essentia) (1.21.5)\n","Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from essentia) (1.15.0)\n","Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from essentia) (3.13)\n","Requirement already satisfied: mir_eval in /usr/local/lib/python3.7/dist-packages (0.7)\n","Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from mir_eval) (1.15.0)\n","Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from mir_eval) (1.4.1)\n","Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from mir_eval) (0.16.0)\n","Requirement already satisfied: numpy>=1.7.0 in /usr/local/lib/python3.7/dist-packages (from mir_eval) (1.21.5)\n"]}],"source":["!pip install essentia\n","!pip install mir_eval"]},{"cell_type":"code","execution_count":2,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":953,"status":"ok","timestamp":1648945799100,"user":{"displayName":"PATRICIO OVALLE","userId":"13329772063495446668"},"user_tz":420},"id":"-t6MK-9Jdg2q","outputId":"d3b84510-f83c-4aee-d527-df0d33d89191"},"outputs":[{"output_type":"stream","name":"stdout","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n","/content/drive/MyDrive/Colab Notebooks/freesound-final\n","['', '/content', '/env/python', '/usr/lib/python37.zip', '/usr/lib/python3.7', '/usr/lib/python3.7/lib-dynload', '/usr/local/lib/python3.7/dist-packages', '/usr/lib/python3/dist-packages', '/usr/local/lib/python3.7/dist-packages/IPython/extensions', '/root/.ipython', '/content/drive/MyDrive/Colab Notebooks/freesound-final']\n"]}],"source":["# Mount drive and cd to notebook folder\n","from google.colab import drive\n","drive.mount('/content/drive')\n","\n","%cd \"/content/drive/MyDrive/Colab Notebooks/freesound-final\"\n","\n","import sys\n","COLAB_WORKDIR = \"/content/drive/MyDrive/Colab Notebooks/freesound-final\"\n","if COLAB_WORKDIR not in sys.path:\n","  sys.path.append(COLAB_WORKDIR)\n","print(sys.path)"]},{"cell_type":"code","execution_count":3,"metadata":{"executionInfo":{"elapsed":2043,"status":"ok","timestamp":1648945801136,"user":{"displayName":"PATRICIO OVALLE","userId":"13329772063495446668"},"user_tz":420},"id":"6Ag-UHBDddIp"},"outputs":[],"source":["import numpy as np\n","import pandas as pd\n","import essentia\n","import essentia.standard as estd\n","import matplotlib.pyplot as plt\n","from IPython.display import display, Audio\n","from mir_eval.sonify import pitch_contour"]},{"cell_type":"code","execution_count":4,"metadata":{"executionInfo":{"elapsed":197,"status":"ok","timestamp":1648945801331,"user":{"displayName":"PATRICIO OVALLE","userId":"13329772063495446668"},"user_tz":420},"id":"s3i5jhmqddIq"},"outputs":[],"source":["# Define the sample rate of the input audio\n","fs = 44100\n","\n","def analyze_sound(audio_path, audio_id=None):\n","  # Load the target audio\n","  audio = estd.MonoLoader(filename=audio_path)()\n","\n","  # Use PitchMelodia from Essentia to analyse the pitch contour of the predominant\n","  # melody and extract the pitch onset times\n","  hop_size = 64\n","  pitch_extractor = estd.PredominantPitchMelodia(frameSize=8820, hopSize=hop_size)\n","  pitch_values, pitch_confidence = pitch_extractor(audio)\n","\n","  # Pitch is estimated on frames. Compute frame time positions.\n","  pitch_times = np.linspace(0.0, len(audio) / fs, len(pitch_values))\n","  pitch_samples = [int(round(t * fs)) for t in pitch_times]\n","\n","  onset_times, durations, notes = estd.PitchContourSegmentation(\n","      hopSize=hop_size,\n","      minDuration=.6,\n","      pitchDistanceThreshold=30,\n","      rmsThreshold=-4\n","  )(pitch_values, audio)\n","\n","  onset_samples = [int(round(t * fs)) for t in onset_times]\n","\n","  frame_start_end_times = zip(onset_times[:-1], onset_times[1:])\n","  frame_start_end_samples = list(zip(onset_samples[:-1], onset_samples[1:]))\n","\n","  analysis_output = []\n","\n","  for ix, (fstart_time, fend_time) in enumerate(frame_start_end_times):\n","      if fend_time - fstart_time > .1:\n","          # Get corresponding audio chunk\n","          fstart, fend = frame_start_end_samples[ix]\n","\n","          try:\n","              frame = audio[fstart:fend]\n","          except:\n","              frame = audio[fstart:]\n","\n","          if len(frame) % 2 != 0:\n","            frame = frame[:-1] # Make frame size even\n","\n","          # Initialize dictionary to store analysis results with some basic metadata\n","          frame_output = {\n","              'freesound_id': audio_id,\n","              'id': f'None_{audio_id}',\n","              'path': audio_path,\n","              'start_sample': fstart,\n","              'end_sample': fend,\n","          }\n","\n","          # Extract loudness\n","          loudness_algo = estd.Loudness()\n","          loudness = loudness_algo(frame)\n","          frame_output['loudness'] = loudness / len(frame)  # Normalize by length of frame\n","          \n","          # Extract MFCC coefficients\n","          w_algo = estd.Windowing(type = 'hann')\n","          spectrum_algo = estd.Spectrum()\n","          mfcc_algo = estd.MFCC()\n","          spec = spectrum_algo(w_algo(frame))\n","          _, mfcc_coeffs = mfcc_algo(spec)\n","          frame_output.update({'mfcc_{0}'.format(j): mfcc_coeffs[j] for j in range(0, len(mfcc_coeffs))})\n","\n","          # Use extracted pitch\n","          pitch_ix = list(onset_times).index(fstart_time)\n","          frame_output['mean_pitch'] = notes[pitch_ix]\n","\n","          # Add frame analysis results to output\n","          analysis_output.append(frame_output)\n","\n","  return analysis_output"]},{"cell_type":"markdown","metadata":{"id":"w9cZSc5PddIq"},"source":["## Analyze source collection"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"q2jZIwLvddIr","outputId":"caaa83a5-eb94-42ff-da4b-0bd79a97d7c1","scrolled":false},"outputs":[{"output_type":"stream","name":"stdout","text":["Analyzing sound with id 56199 [1/100]\n","Analyzing sound with id 56089 [2/100]\n","Analyzing sound with id 48684 [3/100]\n","Analyzing sound with id 337897 [4/100]\n","Analyzing sound with id 81807 [5/100]\n","Analyzing sound with id 508530 [6/100]\n","Analyzing sound with id 56025 [7/100]\n","Analyzing sound with id 55966 [8/100]\n","Analyzing sound with id 55889 [9/100]\n","Analyzing sound with id 55941 [10/100]\n","Analyzing sound with id 557469 [11/100]\n","Analyzing sound with id 92005 [12/100]\n","Analyzing sound with id 107319 [13/100]\n","Analyzing sound with id 197859 [14/100]\n","Analyzing sound with id 361565 [15/100]\n","Analyzing sound with id 333744 [16/100]\n","Analyzing sound with id 61928 [17/100]\n","Analyzing sound with id 332598 [18/100]\n","Analyzing sound with id 494670 [19/100]\n","Analyzing sound with id 494643 [20/100]\n","Analyzing sound with id 494667 [21/100]\n","Analyzing sound with id 502306 [22/100]\n","Analyzing sound with id 355814 [23/100]\n","Analyzing sound with id 494552 [24/100]\n","Analyzing sound with id 494669 [25/100]\n","Analyzing sound with id 494682 [26/100]\n","Analyzing sound with id 494668 [27/100]\n","Analyzing sound with id 528200 [28/100]\n","Analyzing sound with id 153627 [29/100]\n","Analyzing sound with id 494538 [30/100]\n","Analyzing sound with id 621591 [31/100]\n","Analyzing sound with id 494553 [32/100]\n","Analyzing sound with id 494529 [33/100]\n","Analyzing sound with id 494673 [34/100]\n","Analyzing sound with id 494679 [35/100]\n","Analyzing sound with id 494541 [36/100]\n","Analyzing sound with id 494676 [37/100]\n","Analyzing sound with id 494531 [38/100]\n","Analyzing sound with id 494649 [39/100]\n","Analyzing sound with id 4502 [40/100]\n","Analyzing sound with id 621594 [41/100]\n"]}],"source":["DATAFRAME_FILENAME = 'dataframe.csv'  # DataFrame file of the sound source collection to analyze\n","DATAFRAME_SOURCE_FILENAME = 'dataframe_source.csv'  # DataFrame file where to store the results of our analysis\n","\n","# Load the DataFrame of the sound source collection created in previous notebook and analyze all sound files in it\n","df = pd.read_csv(open(DATAFRAME_FILENAME), index_col=0)\n","analyses = []\n","for i in range(0, len(df)):\n","    sound = df.iloc[i]  # Get DataFrame sound at position 'i'\n","    print('Analyzing sound with id {0} [{1}/{2}]'.format(sound['freesound_id'], i + 1, len(df)))\n","     # Split audio in chunks\n","     # Chunk times will be based on the pitch times of the predominant melody when using use_melody=True\n","    analysis_output = analyze_sound(sound['path'], audio_id=sound['freesound_id'])\n","    analyses += analysis_output\n","\n","# Store analysis results in a new Pandas DataFrame and save it\n","df_source = pd.DataFrame(analyses)\n","df_source.to_csv(DATAFRAME_SOURCE_FILENAME)\n","print('Saved source DataFrame with {0} entries! {1}'.format(len(df_source), DATAFRAME_SOURCE_FILENAME))\n","\n","display(df_source)  # Show DataFrame contents\n","df_source.describe()  # Show some statistics of numerical fields in the DataFrame"]},{"cell_type":"markdown","metadata":{"id":"hOUm8X7XddIr"},"source":["## Analyze the target sound file"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"mURJcmLOw7uL"},"outputs":[],"source":["TARGET_SOUND_PATH = 'targets/short_V1bFr2SWP1I.wav'  # Filename of the target audio we'll reconstruct\n","DATAFRAME_TARGET_FILE_FILENAME = 'dataframe_target.csv'  # DataFrame file where to store the analysis results of the target audio\n","\n","# Analyze the target audio file and store results in a new DataFrame\n","print('Analyzing target sound {0}'.format(TARGET_SOUND_PATH))\n","\n","target_analysis = analyze_sound(TARGET_SOUND_PATH)\n","df_target = pd.DataFrame(target_analysis)\n","df_target.to_csv(DATAFRAME_TARGET_FILE_FILENAME)\n","print('Saved target dataframe with {0} entries! {1}'.format(len(df_target), DATAFRAME_TARGET_FILE_FILENAME))\n","\n","audio_path = TARGET_SOUND_PATH\n","\n","# Load the target audio\n","audio = estd.MonoLoader(filename=audio_path)()\n","display(Audio(audio, rate=fs))\n","\n","# Use PitchMelodia from Essentia to analyse the pitch contour of the predominant\n","# melody and extract the pitch onset times\n","pitch_extractor = estd.PredominantPitchMelodia(frameSize=4410, hopSize=64)\n","pitch_values, pitch_confidence = pitch_extractor(audio)\n","\n","# Pitch is estimated on frames. Compute frame time positions.\n","pitch_times = np.linspace(0.0, len(audio) / fs, len(pitch_values))\n","pitch_samples = [int(round(t * fs)) for t in pitch_times]\n","\n","# Generate a sine wave signal following the estimated pitch\n","print('Estimated pitch:')\n","synthesized_melody = pitch_contour(pitch_times, pitch_values, fs).astype(np.float32)[:len(audio)]\n","display(Audio(synthesized_melody, rate=fs))\n","\n","# Convert pitch values from Hz to MIDI notes using the PitchContourSegmentation algorithm\n","onset_times, durations, notes = estd.PitchContourSegmentation(\n","    hopSize=64,\n","    minDuration=.6,\n","    pitchDistanceThreshold=30,\n","    rmsThreshold=-4\n",")(pitch_values, audio)\n","\n","print('Original audio with onset time markers:')\n","marker = estd.AudioOnsetsMarker(onsets=onset_times, type='beep')\n","marked_audio = marker(audio)\n","display(Audio(marked_audio, rate=fs))\n","\n","print('Mix of all signals')\n","display(Audio(marked_audio * 0.5 + synthesized_melody * 0.5, rate=fs))\n","\n","# Plot the estimated pitch contour and confidence over time\n","f, axarr = plt.subplots(2, sharex=True)\n","axarr[0].plot(pitch_samples, pitch_values)\n","axarr[0].set_title('estimated pitch [Hz]')\n","axarr[1].plot(pitch_samples, pitch_confidence)\n","axarr[1].set_title('pitch confidence')\n","f.set_figheight(5)\n","f.set_figwidth(15)\n","\n","# Plot target audio file waveform and show ticks at the start samples of the pitches\n","plt.figure(figsize=(15,5))\n","plt.plot(audio)\n","plt.vlines(df_target['start_sample'].values, -1, 1, color='red')\n","plt.axis([0, len(audio), -1, 1])\n","plt.title('Target audio file')\n","plt.show()\n","\n","display(df_target)\n","df_target.describe()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"FnXQ2EZXHq48"},"outputs":[],"source":[""]}],"metadata":{"colab":{"collapsed_sections":[],"name":"2-analyze-source-collection-and-target.ipynb","provenance":[]},"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.9.10"}},"nbformat":4,"nbformat_minor":0}