debussy_chordified.py

# -*- coding: utf-8 -*-
"""DeBussy_Chordified.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/17eRXNGvoMLAXhlq1oVOJFfUjO4w_tseA

# DeBussy Chordified (ver. 1.0)

***

Powered by tegridy-tools: https://github.com/asigalov61/tegridy-tools

***

Credit for GPT2-RGA code used in this colab goes out @ Sashmark97 https://github.com/Sashmark97/midigen and @ Damon Gwinn https://github.com/gwinndr/MusicTransformer-Pytorch

***

WARNING: This complete implementation is a functioning model of the Artificial Intelligence. Please excercise great humility, care, and respect. https://www.nscai.gov/

***

#### Project Los Angeles

#### Tegridy Code 2022

***

# (Setup Environment)
"""

#@title nvidia-smi gpu check
!nvidia-smi

#@title Install all dependencies (run only once per session)

!git clone https://github.com/asigalov61/DeBussy

!pip install torch

!pip install tqdm
!pip install matplotlib
!pip install torch-summary

!apt install fluidsynth #Pip does not work for some reason. Only apt works
!pip install midi2audio
!pip install pretty_midi

#@title Import all needed modules

print('Loading needed modules. Please wait...')
import os
import random
import copy

from collections import OrderedDict

from tqdm import tqdm

import matplotlib.pyplot as plt

from torchsummary import summary

print('Loading TMIDIX module...')
os.chdir('/content/DeBussy')

import TMIDIX
from GPT2RGAX import *

from midi2audio import FluidSynth
import pretty_midi
import librosa.display
from IPython.display import Audio

os.chdir('/content/')

"""# (MODEL LOAD)"""

# Commented out IPython magic to ensure Python compatibility.
#@title Unzip pre-trained DeBussy Model
# %cd /content/DeBussy/Model

print('=' * 70)
print('Unzipping pre-trained dataset-model...Please wait...')
print('=' * 70)

!cat /content/DeBussy/Model/DeBussy-Trained-Model.zip* > DeBussy-Trained-Model.zip
print('=' * 70)

!unzip -j DeBussy-Trained-Model.zip
print('=' * 70)

print('Done! Enjoy! :)')
print('=' * 70)
# %cd /content/

#@title Load/Reload the model

full_path_to_model_checkpoint = "/content/DeBussy/Model/DeBussy-Trained-Model.pth" #@param {type:"string"}

print('Loading the model...')
config = GPTConfig(640, 
                   512,
                   dim_feedforward=512,
                   n_layer=16, 
                   n_head=16, 
                   n_embd=512,
                   enable_rpr=True,
                   er_len=512)

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

model = GPT(config)

state_dict = torch.load(full_path_to_model_checkpoint, map_location=device)

new_state_dict = OrderedDict()
for k, v in state_dict.items():
    name = k[7:] #remove 'module'
    new_state_dict[name] = v

model.load_state_dict(new_state_dict)

model.to(device)

model.eval()

print('Done!')

summary(model)

"""# (GENERATE)

# Load Custom MIDI / MIDI seed
"""

#@title Custom MIDI option
full_path_to_custom_MIDI = "/content/DeBussy/DeBussy-Seed-1.mid" #@param {type:"string"}

print('Loading custom MIDI file...')

score = TMIDIX.midi2ms_score(open(full_path_to_custom_MIDI, 'rb').read())

events_matrix1 = []

itrack = 1

while itrack < len(score):
    for event in score[itrack]:         
        if event[0] == 'note' and event[3] != 9:
            events_matrix1.append(event)
    itrack += 1

# final processing...

events_matrix1.sort(key=lambda x: x[4], reverse=True) # Sort by pitch H -> L
events_matrix1.sort(key=lambda x: x[1]) # Then sort by start-times

if len(events_matrix1) > 0:

    # recalculating timings

    for e in events_matrix1:
        e[1] = int(e[1] / 10) # Time-shift
        e[2] = int(e[2] / 10) # Duration

    melody_chords = []

    pe = events_matrix1[0]

    cho = []

    for e in events_matrix1:

        if  e[1]-pe[1] == 0:
          cho.append([e[1]-pe[1], e[2], e[4], e[5]])
        else:
          melody_chords.append(cho)
          
          cho = []
        
          cho.append([e[1]-pe[1], e[2], e[4], e[5]])


        pe = e

melody_chords_f = []

for m in melody_chords:
  if len(m) == 1:
    noc = 638
  else:
    noc = 639

  chord = [noc]

  chord.extend([max(1, min(253, m[0][0]))])
  chord.extend([max(1, min(253, m[0][1]))+256])

  for mm in m:
    if mm[2]+512 not in chord:
      chord.append(mm[2]+512)

  melody_chords_f.append(chord)


inputs = []

for m in melody_chords_f:
  inputs.extend(m)


print('Done!')

"""# Continuations"""

#@title Single Continuation Block Generator

#@markdown NOTE: Play with the settings to get different results

number_of_prime_tokens = 128 #@param {type:"slider", min:32, max:384, step:8}
temperature = 0.8 #@param {type:"slider", min:0.1, max:1, step:0.1}
show_stats = True #@param {type:"boolean"}

#===================================================================
print('=' * 70)
print('DeBussy Music Model Continuation Generator')
print('=' * 70)

print('Generation settings:')
print('=' * 70)
print('Number of prime tokens:', number_of_prime_tokens)
print('Model temperature:', temperature)

print('=' * 70)
print('Generating...')


rand_seq = model.generate(torch.Tensor(inputs[:number_of_prime_tokens]), 
                                          target_seq_length=512,
                                          temperature=temperature,
                                          stop_token=640,
                                          verbose=show_stats)
  
out1 = rand_seq[0].cpu().numpy().tolist()


if len(out1) != 0:
    
    song = out1
    song_f = []
    time = 0
    dur = 0
    vel = 0
    pitch = 0
    channel = 0
    son = [song[0]]
    for s in song[1:]:

        if s < 638:
          son.append(s)

        else:
          if len(son) > 2:

            time += son[1] * 10

            dur = (son[2] - 256) * 10
            
            channel = 0 # Piano

            if son[0] == 638:

                vel = 80
            else:
                vel = 110

            for ss in son[3:]:
              pitch = ss - 512
                              
              song_f.append(['note', time, dur, channel, pitch, vel ])
            
            son = []
            son.append(s)

    detailed_stats = TMIDIX.Tegridy_SONG_to_MIDI_Converter(song_f,
                                                        output_signature = 'DeBussy',  
                                                        output_file_name = '/content/DeBussy-Music-Composition', 
                                                        track_name='Project Los Angeles',
                                                        list_of_MIDI_patches=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                                                        number_of_ticks_per_quarter=500)

    print('Done!')


print('Displaying resulting composition...')
fname = '/content/DeBussy-Music-Composition'

pm = pretty_midi.PrettyMIDI(fname + '.mid')

# Retrieve piano roll of the MIDI file
piano_roll = pm.get_piano_roll()

plt.figure(figsize=(14, 5))
librosa.display.specshow(piano_roll, x_axis='time', y_axis='cqt_note', fmin=1, hop_length=160, sr=16000, cmap=plt.cm.hot)
plt.title(fname)

FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
Audio(str(fname + '.wav'), rate=16000)

#@title Multiple Continuation Blocks Generator

#@markdown NOTE: Play with the settings to get different results

number_of_prime_tokens = 128 #@param {type:"slider", min:32, max:384, step:8}
number_of_continuation_blocks = 5 #@param {type:"slider", min:1, max:100, step:1}
temperature = 0.8 #@param {type:"slider", min:0.1, max:1, step:0.1}
show_stats = False #@param {type:"boolean"}

#===================================================================
print('=' * 70)
print('DeBussy Music Model Continuation Generator')
print('=' * 70)

print('Generation settings:')
print('=' * 70)
print('Number of prime tokens:', number_of_prime_tokens)
print('Number of continuation blocks:', number_of_continuation_blocks)
print('Model temperature:', temperature)

print('=' * 70)
print('Generating...')

out1 = []

out1 = inputs[:number_of_prime_tokens]

for i in tqdm(range(number_of_continuation_blocks)):

  rand_seq = model.generate(torch.Tensor(out1[-number_of_prime_tokens:]), 
                                            target_seq_length=512,
                                            temperature=temperature,
                                            stop_token=640,
                                            verbose=show_stats)
    
  out = rand_seq[0].cpu().numpy().tolist()

  out1.extend(out[-(512-number_of_prime_tokens):])

if len(out1) != 0:
  
  song = out1
  song_f = []
  time = 0
  dur = 0
  vel = 0
  pitch = 0
  channel = 0
  son = [song[0]]
  for s in song[1:]:

      if s < 638:
        son.append(s)

      else:
        if len(son) > 2:

          time += son[1] * 10

          dur = (son[2] - 256) * 10
          
          channel = 0 # Piano

          if son[0] == 638:

              vel = 80
          else:
              vel = 110

          for ss in son[3:]:
            pitch = ss - 512
                            
            song_f.append(['note', time, dur, channel, pitch, vel ])
          
          son = []
          son.append(s)

  detailed_stats = TMIDIX.Tegridy_SONG_to_MIDI_Converter(song_f,
                                                      output_signature = 'DeBussy',  
                                                      output_file_name = '/content/DeBussy-Music-Composition', 
                                                      track_name='Project Los Angeles',
                                                      list_of_MIDI_patches=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                                                      number_of_ticks_per_quarter=500)

  print('Done!')


print('Displaying resulting composition...')
fname = '/content/DeBussy-Music-Composition'

pm = pretty_midi.PrettyMIDI(fname + '.mid')

# Retrieve piano roll of the MIDI file
piano_roll = pm.get_piano_roll()

plt.figure(figsize=(14, 5))
librosa.display.specshow(piano_roll, x_axis='time', y_axis='cqt_note', fmin=1, hop_length=160, sr=16000, cmap=plt.cm.hot)
plt.title(fname)

FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
Audio(str(fname + '.wav'), rate=16000)

"""# Inpainting / Controlled Generation"""

#@title Custom MIDI Inpaiting / Controlled Generator

#@markdown NOTE: Play with the settings to get different results

control_type = "Note-Chord" #@param ["Note-Chord", "Note-Chord-Time", "Note-Chord-Time-Duration", "Note-Chord-Time-Duration-Melody-Pitch"]
temperature = 0.8 #@param {type:"slider", min:0.1, max:1, step:0.1}
show_stats = False #@param {type:"boolean"}

#===================================================================
print('=' * 70)
print('DeBussy Music Model Inpainting / Controlled Generator')
print('=' * 70)

print('Generation settings:')
print('=' * 70)
print('Control type:', control_type)
print('Model temperature:', temperature)

print('=' * 70)
print('Generating...')


if control_type == 'Note-Chord':
  ctrl = 1

if control_type == 'Note-Chord-Time':
  ctrl = 2

if control_type == 'Note-Chord-Time-Duration':
  ctrl = 3

if control_type == 'Note-Chord-Time-Duration-Melody-Pitch':
  ctrl = 4


out1 = melody_chords_f[0][:ctrl]

for i in tqdm(range(1, len(melody_chords_f))):

  rand_seq = model.generate(torch.Tensor(out1[-500:]), 
                                            target_seq_length=512,
                                            temperature=temperature,
                                            stop_token=638,
                                            verbose=show_stats)
    
  out = rand_seq[0].cpu().numpy().tolist()

  out1.extend(out[len(out1[-500:]):])
  out1.extend(melody_chords_f[i][:ctrl])

if len(out1) != 0:
  
  song = out1
  song_f = []
  time = 0
  dur = 0
  vel = 0
  pitch = 0
  channel = 0
  son = [song[0]]
  for s in song[1:]:

      if s < 638:
        son.append(s)

      else:
        if len(son) > 2:

          time += son[1] * 10

          dur = (son[2] - 256) * 10
          
          channel = 0 # Piano

          if son[0] == 638:

              vel = 80
          else:
              vel = 110

          for ss in son[3:]:
            pitch = ss - 512
                            
            song_f.append(['note', time, dur, channel, pitch, vel ])
          
          son = []
          son.append(s)

  detailed_stats = TMIDIX.Tegridy_SONG_to_MIDI_Converter(song_f,
                                                      output_signature = 'DeBussy',  
                                                      output_file_name = '/content/DeBussy-Music-Composition', 
                                                      track_name='Project Los Angeles',
                                                      list_of_MIDI_patches=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                                                      number_of_ticks_per_quarter=500)

  print('Done!')


print('Displaying resulting composition...')
fname = '/content/DeBussy-Music-Composition'

pm = pretty_midi.PrettyMIDI(fname + '.mid')

# Retrieve piano roll of the MIDI file
piano_roll = pm.get_piano_roll()

plt.figure(figsize=(14, 5))
librosa.display.specshow(piano_roll, x_axis='time', y_axis='cqt_note', fmin=1, hop_length=160, sr=16000, cmap=plt.cm.hot)
plt.title(fname)

FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
Audio(str(fname + '.wav'), rate=16000)

"""# Notes / Chords Progressions"""

#@title Notes/Chords Progressions Generator

#@markdown NOTE: Play with the settings to get different results
progression_type = "Notes-Progression" #@param ["Notes-Progression", "Chords-Progression"]
control_type = "Note-Chord" #@param ["Note-Chord", "Note-Chord-Time", "Note-Chord-Time-Duration", "Note-Chord-Time-Duration-Melody-Pitch"]
desired_time = 25 #@param {type:"slider", min:1, max:255, step:1}
desired_duration = 25 #@param {type:"slider", min:1, max:255, step:1}
desired_melody_pitch = 75 #@param {type:"slider", min:1, max:125, step:1}
temperature = 0.8 #@param {type:"slider", min:0.1, max:1, step:0.1}
show_stats = False #@param {type:"boolean"}


#===================================================================
print('=' * 70)
print('DeBussy Music Model Notes/Chords Progressions Generator')
print('=' * 70)

print('Generation settings:')
print('=' * 70)
print('Progression Type:', progression_type)
print('Control Type:', control_type)
print('Model temperature:', temperature)

print('=' * 70)
print('Generating...')

ctrl = []

if progression_type == 'Notes-Progression':
  ctrl = [638]
else:
  ctrl = [639]

if control_type == 'Note-Chord':
  pass

if control_type == 'Note-Chord-Time':
  ctrl.extend([desired_time])

if control_type == 'Note-Chord-Time-Duration':
  ctrl.extend([desired_time])
  ctrl.extend([desired_duration+256])

if control_type == 'Note-Chord-Time-Duration-Melody-Pitch':
  ctrl.extend([desired_time])
  ctrl.extend([desired_duration+256])
  ctrl.extend([desired_melody_pitch+512])

out1 = copy.deepcopy(ctrl)

for i in tqdm(range(128)):

  try: 
    rand_seq = model.generate(torch.Tensor(out1), 
                                              target_seq_length=512,
                                              temperature=temperature,
                                              stop_token=638,
                                              verbose=show_stats)
      
    out = rand_seq[0].cpu().numpy().tolist()

    out1.extend(out[len(out1):])
    out1.extend(ctrl)

  except:
    break

if len(out1) != 0:
  
  song = out1
  song_f = []
  time = 0
  dur = 0
  vel = 0
  pitch = 0
  channel = 0
  son = [song[0]]
  for s in song[1:]:

      if s < 638:
        son.append(s)

      else:
        if len(son) > 2:

          time += son[1] * 10

          dur = (son[2] - 256) * 10
          
          channel = 0 # Piano

          if son[0] == 638:

              vel = 80
          else:
              vel = 110

          for ss in son[3:]:
            pitch = ss - 512
                            
            song_f.append(['note', time, dur, channel, pitch, vel ])
          
          son = []
          son.append(s)

  detailed_stats = TMIDIX.Tegridy_SONG_to_MIDI_Converter(song_f,
                                                      output_signature = 'DeBussy',  
                                                      output_file_name = '/content/DeBussy-Music-Composition', 
                                                      track_name='Project Los Angeles',
                                                      list_of_MIDI_patches=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                                                      number_of_ticks_per_quarter=500)

  print('Done!')


print('Displaying resulting composition...')
fname = '/content/DeBussy-Music-Composition'

pm = pretty_midi.PrettyMIDI(fname + '.mid')

# Retrieve piano roll of the MIDI file
piano_roll = pm.get_piano_roll()

plt.figure(figsize=(14, 5))
librosa.display.specshow(piano_roll, x_axis='time', y_axis='cqt_note', fmin=1, hop_length=160, sr=16000, cmap=plt.cm.hot)
plt.title(fname)

FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
Audio(str(fname + '.wav'), rate=16000)

"""# Congrats! You did it! :)"""