yoda.py

# -*- coding: utf-8 -*-
"""Yoda.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/17Nw5bpD4C-lSnxStbDAtonVpqtggRRWZ

# Yoda (ver. 1.0)

***

Powered by tegridy-tools TMIDIX Optimus Processors: https://github.com/asigalov61/tegridy-tools

***

Credit for GPT2-RGA code used in this colab goes out @ Sashmark97 https://github.com/Sashmark97/midigen and @ Damon Gwinn https://github.com/gwinndr/MusicTransformer-Pytorch

***

WARNING: This complete implementation is a functioning model of the Artificial Intelligence. Please excercise great humility, care, and respect. https://www.nscai.gov/

***

#### Project Los Angeles

#### Tegridy Code 2022

***

# (Setup Environment)
"""

#@title nvidia-smi gpu check
!nvidia-smi

#@title Install all dependencies (run only once per session)

!git clone https://github.com/asigalov61/Yoda
!pip install torch
!pip install tqdm
!pip install matplotlib

!apt install fluidsynth #Pip does not work for some reason. Only apt works
!pip install midi2audio
!pip install pretty_midi

#@title Import all needed modules

print('Loading needed modules. Please wait...')
import os
import copy
import tqdm as tqdm
import random

print('Loading TMIDIX module...')
os.chdir('/content/Yoda')
import TMIDIX
from GPT2RGAX import *

print('Loading aux modules...')
import matplotlib.pyplot as plt

from midi2audio import FluidSynth
import pretty_midi
import librosa.display
from IPython.display import display, Javascript, HTML, Audio

os.chdir('/content/')
print('Done! Enjoy! :)')

"""# (MODEL)"""

# Commented out IPython magic to ensure Python compatibility.
#@title Unzip pre-trained Yoda model and the training data file
# %cd /content/

print('=' * 70)
print('Unzipping pre-trained dataset-model...Please wait...')
print('=' * 70)

!cat /content/Yoda/Model/Yoda-Trained-Model.zip* > Yoda-Trained-Model.zip
print('=' * 70)

!unzip -j Yoda-Trained-Model.zip
print('=' * 70)

print('Done! Enjoy! :)')
print('=' * 70)
# %cd /content/

"""# (LOAD)"""

#@title Load/Reload the model

from collections import OrderedDict

full_path_to_model_checkpoint = "/content/Yoda-Trained-Model.pth" #@param {type:"string"}

print('Loading the model...')
config = GPTConfig(21938, 
                   1024,
                   dim_feedforward=512,
                   n_layer=8, 
                   n_head=8, 
                   n_embd=512,
                   enable_rpr=True,
                   er_len=1024)

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

model = GPT(config)

state_dict = torch.load(full_path_to_model_checkpoint, map_location=device)

new_state_dict = OrderedDict()
for k, v in state_dict.items():
    name = k[7:] #remove 'module'
    new_state_dict[name] = v

model.load_state_dict(new_state_dict)

model.to(device)

model.eval()

print('Done!')

#@title Load and prep the original training data which will be used to prime the model
full_path_to_original_training_data = "/content/Yoda_Training_Data" #@param {type:"string"}

melody_chords_f = TMIDIX.Tegridy_Any_Pickle_File_Reader(full_path_to_original_training_data)

randomize_dataset = True

print('=' * 70)
print('Prepping INTs dataset...')

if randomize_dataset:
    print('=' * 70)
    print('Randomizing the dataset...')
    random.shuffle(melody_chords_f)
    print('Done!')
    
print('=' * 70)
print('Processing the dataset...')

r = 0

train_data1 = []

itimes = []
ipitches = []


for chords_list in tqdm(melody_chords_f):
    
    train_data1.extend([0]) # Intro/Zero Token
    
    for i in chords_list:

        if i[0] != 0: # This is the chordification line
            train_data1.extend([i[0]]) # start-times
            itimes.extend([i[0]])
            ipitches.extend([i[1] + (i[2] * 16) + (i[3] * 16 * 128)])
        # And this is the main MIDI note line (triple stack)
        main_note = [i[1] + (i[2] * 16) + (i[3] * 16 * 128)] # Main note == [duration / pitch / channel]
        
        if main_note != [0]: # Main note error control...
            train_data1.extend(main_note) # Main note == [duration / pitch / channel]

print('Done!')        
print('=' * 70)
        
print('Total INTs:', len(train_data1))
print('Minimum INT:', min(train_data1))
print('Maximum INT:', max(train_data1))
print('Unique INTs:', len(set(train_data1)))
print('Intro/Zero INTs:', train_data1.count(0))
print('=' * 70)

"""# (GENERATE MUSIC)

### NOTE: Due to the size of the model and the dataset, the output will be (at best) a remix of the original compositions. Only large-scale models trained on huge datasets (like MuseNet) can produce original compositions.
"""

#@title Generate music

#@markdown NOTE: Play with the settings to get different results


priming_type = "Random Intro" #@param ["Random Intro", "Random Point"]
freeze_priming_point = False #@param {type:"boolean"}
number_of_instruments = 11 # DO NOT REDUCE for optimal performance !!!
number_of_prime_tokens = 32 #@param {type:"slider", min:4, max:32, step:4}
number_of_tokens_to_generate = 64 #@param {type:"slider", min:64, max:128, step:16}
number_of_continuation_blocks = 40 #@param {type:"slider", min:10, max:100, step:5}
temperature = 0.8 #@param {type:"slider", min:0.1, max:1, step:0.1}
show_stats = False #@param {type:"boolean"}

#===================================================================
print('=' * 70)
print('Yoda Music Model Continuation Generator')
print('=' * 70)

print('Generation settings:')
print('=' * 70)
print('Priming type:', priming_type)
print('Number of instruments:', number_of_instruments)
print('Number of prime tokens:', number_of_prime_tokens)
print('Number of tokens:', number_of_tokens_to_generate)
print('Number of continuation blocks:', number_of_continuation_blocks)
print('Model temperature:', temperature)

print('=' * 70)
print('Prepping...')

if not freeze_priming_point:
  r = random.randint(0, len(train_data1))

if priming_type == 'Random Intro':
  idx = r+train_data1[r:].index(0)
  out = train_data1[idx:idx+number_of_prime_tokens]
else:
  out = train_data1[r:r+number_of_prime_tokens]

out1 = []

tokens_range = (128 * 16 * number_of_instruments)

print('=' * 70)
print('Generating...')

for i in range(number_of_continuation_blocks):

  if show_stats: 
    print('=' * 70)
    print('Block #', i)

  rand_seq = model.generate(torch.Tensor(out[-number_of_prime_tokens:]), 
                                          target_seq_length=number_of_tokens_to_generate,
                                          temperature=temperature,
                                          stop_token=tokens_range,
                                          verbose=show_stats)
  
  out = rand_seq[0].cpu().numpy().tolist()
  
  out1.extend(out[number_of_prime_tokens:])

print('=' * 70)
print('Done!')

if show_stats:
  print('=' * 70)
  print('Detokenizing output...')

if len(out1) != 0:
    song = []
    song = out1
    song_f = []
    time = 0
    dur = 0
    vel = 0
    pitch = 0
    channel = 0
    
    for s in song:
        if s < 256:
            time += s * 16
            
        else:
            channel = s // 16 // 128

            pitch = (s // 16) % 128
            
            dur = ((s % 16) * 128) + 128
            
            # Velocities for each channel:
            if channel == 0:  # Piano     
                vel = 60
            if channel == 1:  # Guitar     
                vel = 70            
            if channel == 2:  # Bass     
                vel = 60            
            if channel == 3:  # Violin
                vel = 90            
            if channel == 4:  # Cello     
                vel = 100
            if channel == 5:  # Harp     
                vel = 80
            if channel == 6:  # Trumpet     
                vel = 100            
            if channel == 7:  # Clarinet     
                vel = 100           
            if channel == 8:  # Flute
                vel = 100                          
            if channel == 9:  # Drums
                vel = 80            
            if channel == 10:  # Choir     
                vel = 110                  
                               
            song_f.append(['note', time, dur, channel, pitch, vel ])
    
    if show_stats:
      print('=' * 70) # Converting to MIDI
      
    detailed_stats = TMIDIX.Tegridy_SONG_to_MIDI_Converter(song_f,
                                                        output_signature = 'Yoda',  
                                                        output_file_name = '/content/Yoda-Music-Composition', 
                                                        track_name='Project Los Angeles',
                                                        list_of_MIDI_patches=[0, 24, 32, 40, 42, 46, 56, 71, 73, 0, 53, 0, 0, 0, 0, 0],
                                                        number_of_ticks_per_quarter=500)

    if show_stats:
      print('=' * 70)
      print('Detailed MIDI stats:')
      for key, value in detailed_stats.items():
            print('=' * 70)
            print(key, '|', value)

else:
  print('Models output is empty! Check the code...')
  print('Shutting down...')


print('=' * 70)
print('Displaying resulting composition...')
fname = 'Yoda-Music-Composition'

pm = pretty_midi.PrettyMIDI(fname + '.mid')

# Retrieve piano roll of the MIDI file
piano_roll = pm.get_piano_roll()

plt.figure(figsize=(14, 5))
librosa.display.specshow(piano_roll, x_axis='time', y_axis='cqt_note', fmin=1, hop_length=160, sr=16000, cmap=plt.cm.hot)
plt.title(fname)

FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
Audio(str(fname + '.wav'), rate=16000)

"""# Congrats! You did it! :)"""