Los_Angeles_Music_Composer_TTM_Edition.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Los Angeles Music Composer TTM Edition (ver. 4.0)\n",
        "\n",
        "***\n",
        "\n",
        "Powered by tegridy-tools: https://github.com/asigalov61/tegridy-tools\n",
        "\n",
        "***\n",
        "\n",
        "WARNING: This complete implementation is a functioning model of the Artificial Intelligence. Please excercise great humility, care, and respect. https://www.nscai.gov/\n",
        "\n",
        "***\n",
        "\n",
        "#### Project Los Angeles\n",
        "\n",
        "#### Tegridy Code 2023\n",
        "\n",
        "***"
      ],
      "metadata": {
        "id": "gpy3qsulqHa5"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (GPU CHECK)"
      ],
      "metadata": {
        "id": "W_So4w8fqPGL"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "X3rABEpKCO02",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title NVIDIA GPU check\n",
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (SETUP ENVIRONMENT)"
      ],
      "metadata": {
        "id": "C0XxnXGFqVyh"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vK40g6V_BTNj",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Install dependencies\n",
        "!git clone --depth 1 https://github.com/asigalov61/Los-Angeles-Music-Composer\n",
        "!pip install torch\n",
        "!pip install einops\n",
        "!pip install fuzzywuzzy[speedup]\n",
        "!pip install torch-summary\n",
        "!pip install tqdm\n",
        "!pip install matplotlib\n",
        "!apt install fluidsynth #Pip does not work for some reason. Only apt works\n",
        "!pip install midi2audio"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DzCOZU_gBiQV",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Import modules\n",
        "\n",
        "print('=' * 70)\n",
        "print('Loading core Los Angeles Music Composer modules...')\n",
        "\n",
        "import os\n",
        "import pickle\n",
        "import random\n",
        "import secrets\n",
        "import statistics\n",
        "from time import time\n",
        "import tqdm\n",
        "\n",
        "print('=' * 70)\n",
        "print('Loading main Los Angeles Music Composer modules...')\n",
        "import torch\n",
        "\n",
        "%cd /content/Los-Angeles-Music-Composer\n",
        "\n",
        "import TMIDIX\n",
        "from lwa_transformer import *\n",
        "\n",
        "%cd /content/\n",
        "\n",
        "from fuzzywuzzy import process\n",
        "\n",
        "print('=' * 70)\n",
        "print('Loading aux Los Angeles Music Composer modeules...')\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from torchsummary import summary\n",
        "from sklearn import metrics\n",
        "\n",
        "from midi2audio import FluidSynth\n",
        "from IPython.display import Audio, display\n",
        "\n",
        "print('=' * 70)\n",
        "print('Done!')\n",
        "print('Enjoy! :)')\n",
        "print('=' * 70)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eI3aQtHzqSnp"
      },
      "source": [
        "# (LOAD MODEL)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Unzip Pre-Trained Los Angeles Music Composer Model\n",
        "print('=' * 70)\n",
        "%cd /content/Los-Angeles-Music-Composer/Model\n",
        "\n",
        "print('=' * 70)\n",
        "print('Unzipping pre-trained Los Angeles Music Composer model...Please wait...')\n",
        "\n",
        "!cat /content/Los-Angeles-Music-Composer/Model/Los_Angeles_Music_Composer_Trained_Model.zip* > /content/Los-Angeles-Music-Composer/Model/Los_Angeles_Music_Composer_Trained_Model.zip\n",
        "print('=' * 70)\n",
        "\n",
        "!unzip -j /content/Los-Angeles-Music-Composer/Model/Los_Angeles_Music_Composer_Trained_Model.zip\n",
        "print('=' * 70)\n",
        "\n",
        "print('Done! Enjoy! :)')\n",
        "print('=' * 70)\n",
        "%cd /content/\n",
        "print('=' * 70)"
      ],
      "metadata": {
        "cellView": "form",
        "id": "SqbxcFYVq63r"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rDquonbXC2je",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Load Los Angeles Music Composer Model\n",
        "full_path_to_model_checkpoint = \"/content/Los-Angeles-Music-Composer/Model/Los_Angeles_Music_Composer_Model_88835_steps_0.643_loss.pth\" #@param {type:\"string\"}\n",
        "\n",
        "#@markdown Model precision option\n",
        "\n",
        "model_precision = \"bfloat16\" # @param [\"bfloat16\", \"float16\", \"float32\"]\n",
        "\n",
        "#@markdown bfloat16 == Third precision/triple speed (if supported, otherwise the model will default to float16)\n",
        "\n",
        "#@markdown float16 == Half precision/double speed\n",
        "\n",
        "#@markdown float32 == Full precision/normal speed\n",
        "\n",
        "plot_tokens_embeddings = False # @param {type:\"boolean\"}\n",
        "\n",
        "print('=' * 70)\n",
        "print('Loading Los Angeles Music Composer Pre-Trained Model...')\n",
        "print('Please wait...')\n",
        "print('=' * 70)\n",
        "print('Instantiating model...')\n",
        "\n",
        "torch.backends.cuda.matmul.allow_tf32 = True # allow tf32 on matmul\n",
        "torch.backends.cudnn.allow_tf32 = True # allow tf32 on cudnn\n",
        "device_type = 'cuda'\n",
        "\n",
        "if model_precision == 'bfloat16' and torch.cuda.is_bf16_supported():\n",
        "  dtype = 'bfloat16'\n",
        "else:\n",
        "  dtype = 'float16'\n",
        "\n",
        "if model_precision == 'float16':\n",
        "  dtype = 'float16'\n",
        "\n",
        "if model_precision == 'float32':\n",
        "  dtype = 'float32'\n",
        "\n",
        "ptdtype = {'float32': torch.float32, 'bfloat16': torch.bfloat16, 'float16': torch.float16}[dtype]\n",
        "ctx = torch.amp.autocast(device_type=device_type, dtype=ptdtype)\n",
        "\n",
        "SEQ_LEN = 4096\n",
        "\n",
        "# instantiate the model\n",
        "\n",
        "model = LocalTransformer(\n",
        "    num_tokens = 2831,\n",
        "    dim = 1024,\n",
        "    depth = 36,\n",
        "    causal = True,\n",
        "    local_attn_window_size = 512,\n",
        "    max_seq_len = SEQ_LEN\n",
        ")\n",
        "\n",
        "model = torch.nn.DataParallel(model)\n",
        "\n",
        "model.cuda()\n",
        "print('=' * 70)\n",
        "\n",
        "print('Loading model checkpoint...')\n",
        "\n",
        "model.load_state_dict(torch.load(full_path_to_model_checkpoint))\n",
        "print('=' * 70)\n",
        "\n",
        "model.eval()\n",
        "\n",
        "print('Done!')\n",
        "print('=' * 70)\n",
        "\n",
        "print('Model will use', dtype, 'precision...')\n",
        "print('=' * 70)\n",
        "\n",
        "# Model stats\n",
        "print('Model summary...')\n",
        "summary(model)\n",
        "\n",
        "# Plot Token Embeddings\n",
        "\n",
        "if plot_tokens_embeddings:\n",
        "  tok_emb = model.module.token_emb.weight.detach().cpu().tolist()\n",
        "\n",
        "  cos_sim = metrics.pairwise_distances(\n",
        "    tok_emb, metric='cosine'\n",
        "  )\n",
        "  plt.figure(figsize=(7, 7))\n",
        "  plt.imshow(cos_sim, cmap=\"inferno\", interpolation=\"nearest\")\n",
        "  im_ratio = cos_sim.shape[0] / cos_sim.shape[1]\n",
        "  plt.colorbar(fraction=0.046 * im_ratio, pad=0.04)\n",
        "  plt.xlabel(\"Position\")\n",
        "  plt.ylabel(\"Position\")\n",
        "  plt.tight_layout()\n",
        "  plt.plot()\n",
        "  plt.savefig(\"/content/Los-Angeles-Music-Composer-Tokens-Embeddings-Plot.png\", bbox_inches=\"tight\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (LOAD AUX DATA)"
      ],
      "metadata": {
        "id": "-u40ZSHaJhZR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Unzip Los Angeles Music Composer Aux Data\n",
        "print('=' * 70)\n",
        "%cd /content/Los-Angeles-Music-Composer/Aux-Data\n",
        "\n",
        "print('=' * 70)\n",
        "print('Unzipping Los Angeles Music Composer Aux Data...Please wait...')\n",
        "\n",
        "!cat /content/Los-Angeles-Music-Composer/Aux-Data/Los_Angeles_Music_Composer_Aux_Data.zip* > /content/Los-Angeles-Music-Composer/Aux-Data/Los_Angeles_Music_Composer_Aux_Data.zip\n",
        "print('=' * 70)\n",
        "\n",
        "!unzip -j /content/Los-Angeles-Music-Composer/Aux-Data/Los_Angeles_Music_Composer_Aux_Data.zip\n",
        "print('=' * 70)\n",
        "\n",
        "print('Done! Enjoy! :)')\n",
        "print('=' * 70)\n",
        "%cd /content/\n",
        "print('=' * 70)"
      ],
      "metadata": {
        "cellView": "form",
        "id": "JkAG_70WJoLF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Load Los Angeles Music Composer Aux Data\n",
        "\n",
        "AUX_DATA = TMIDIX.Tegridy_Any_Pickle_File_Reader('/content/Los-Angeles-Music-Composer/Aux-Data/Los_Angeles_Music_Composer_Aux_Data')\n",
        "\n",
        "print('Done!')"
      ],
      "metadata": {
        "cellView": "form",
        "id": "j44lPUnvKB1E"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (GENERATE)"
      ],
      "metadata": {
        "id": "7xNyANjZsCOi"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dkvXYwR_qSnx",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Standard/Simple Continuation\n",
        "\n",
        "#@markdown Text-To-Music Settings\n",
        "\n",
        "#@markdown NOTE: You can enter any desired title or artist, or both\n",
        "\n",
        "enter_desired_song_title = \"Family Guy\" #@param {type:\"string\"}\n",
        "enter_desired_artist = \"TV Themes\" #@param {type:\"string\"}\n",
        "\n",
        "#@markdown Generation Settings\n",
        "\n",
        "number_of_tokens_to_generate = 512 #@param {type:\"slider\", min:32, max:2048, step:32}\n",
        "number_of_batches_to_generate = 4 #@param {type:\"slider\", min:1, max:16, step:1}\n",
        "temperature = 0.9 #@param {type:\"slider\", min:0.1, max:1, step:0.1}\n",
        "allow_model_to_stop_generation_if_needed = False #@param {type:\"boolean\"}\n",
        "render_MIDI_to_audio = True # @param {type:\"boolean\"}\n",
        "\n",
        "print('=' * 70)\n",
        "print('Los Angeles Music Composer TTM Model Generator')\n",
        "print('=' * 70)\n",
        "\n",
        "print('Searching titles...Please wait...')\n",
        "random.shuffle(AUX_DATA)\n",
        "\n",
        "titles_index = []\n",
        "\n",
        "for A in AUX_DATA:\n",
        "  titles_index.append(A[0])\n",
        "\n",
        "search_string = ''\n",
        "\n",
        "if enter_desired_song_title != '' and enter_desired_artist != '':\n",
        "  search_string = enter_desired_song_title + ' --- ' + enter_desired_artist\n",
        "\n",
        "else:\n",
        "  search_string = enter_desired_song_title + enter_desired_artist\n",
        "\n",
        "search_match = process.extract(query=search_string, choices=titles_index, limit=1)\n",
        "search_index = titles_index.index(search_match[0][0])\n",
        "\n",
        "print('Done!')\n",
        "print('=' * 70)\n",
        "print('Selected title:', AUX_DATA[search_index][0])\n",
        "print('=' * 70)\n",
        "\n",
        "if allow_model_to_stop_generation_if_needed:\n",
        "  min_stop_token = 2816\n",
        "else:\n",
        "  min_stop_token = 0\n",
        "\n",
        "outy = AUX_DATA[search_index][1]\n",
        "\n",
        "block_marker = sum([(y * 10) for y in outy if y < 128]) / 1000\n",
        "\n",
        "inp = [outy] * number_of_batches_to_generate\n",
        "\n",
        "inp = torch.LongTensor(inp).cuda()\n",
        "\n",
        "with ctx:\n",
        "  out = model.module.generate(inp,\n",
        "                        number_of_tokens_to_generate,\n",
        "                        temperature=temperature,\n",
        "                        return_prime=True,\n",
        "                        min_stop_token=min_stop_token,\n",
        "                        verbose=True)\n",
        "\n",
        "out0 = out.tolist()\n",
        "print('=' * 70)\n",
        "print('Done!')\n",
        "print('=' * 70)\n",
        "#======================================================================\n",
        "print('Rendering results...')\n",
        "\n",
        "for i in range(number_of_batches_to_generate):\n",
        "\n",
        "  print('=' * 70)\n",
        "  print('Batch #', i)\n",
        "  print('=' * 70)\n",
        "\n",
        "  out1 = out0[i]\n",
        "\n",
        "  print('Sample INTs', out1[:12])\n",
        "  print('=' * 70)\n",
        "\n",
        "  if len(out) != 0:\n",
        "\n",
        "      song = out1\n",
        "      song_f = []\n",
        "      tim = 0\n",
        "      dur = 0\n",
        "      vel = 0\n",
        "      pitch = 0\n",
        "      channel = 0\n",
        "\n",
        "      son = []\n",
        "      song1 = []\n",
        "\n",
        "      for s in song:\n",
        "        if s >= 128 and s < (12*128)+1152:\n",
        "          son.append(s)\n",
        "        else:\n",
        "          if len(son) == 3:\n",
        "            song1.append(son)\n",
        "          son = []\n",
        "          son.append(s)\n",
        "\n",
        "      for ss in song1:\n",
        "\n",
        "        tim += ss[0] * 10\n",
        "\n",
        "        dur = ((ss[1]-128) // 8) * 20\n",
        "        vel = (((ss[1]-128) % 8)+1) * 15\n",
        "\n",
        "        channel = (ss[2]-1152) // 128\n",
        "        pitch = (ss[2]-1152) % 128\n",
        "\n",
        "        song_f.append(['note', tim, dur, channel, pitch, vel ])\n",
        "\n",
        "      detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(song_f,\n",
        "                                                                output_signature = 'Los Angeles Music Composer',\n",
        "                                                                output_file_name = '/content/Los-Angeles-Music-Composer-Music-Composition_'+str(i),\n",
        "                                                                track_name='Project Los Angeles',\n",
        "                                                                list_of_MIDI_patches=[0, 24, 32, 40, 42, 46, 56, 71, 73, 0, 53, 19, 0, 0, 0, 0]\n",
        "                                                                )\n",
        "      print('=' * 70)\n",
        "      print('Displaying resulting composition...')\n",
        "      print('=' * 70)\n",
        "\n",
        "      fname = '/content/Los-Angeles-Music-Composer-Music-Composition_'+str(i)\n",
        "\n",
        "      x = []\n",
        "      y =[]\n",
        "      c = []\n",
        "\n",
        "      colors = ['red', 'yellow', 'green', 'cyan', 'blue', 'pink', 'orange', 'purple', 'gray', 'white', 'gold', 'silver']\n",
        "\n",
        "      for s in song_f:\n",
        "        x.append(s[1] / 1000)\n",
        "        y.append(s[4])\n",
        "        c.append(colors[s[3]])\n",
        "\n",
        "      if render_MIDI_to_audio:\n",
        "        FluidSynth(\"/usr/share/sounds/sf2/FluidR3_GM.sf2\", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))\n",
        "        display(Audio(str(fname + '.wav'), rate=16000))\n",
        "\n",
        "      plt.figure(figsize=(14,5))\n",
        "      ax=plt.axes(title=fname)\n",
        "      ax.set_facecolor('black')\n",
        "\n",
        "      plt.scatter(x,y, c=c)\n",
        "\n",
        "      ax.axvline(x=block_marker, c='w')\n",
        "\n",
        "      plt.xlabel(\"Time\")\n",
        "      plt.ylabel(\"Pitch\")\n",
        "      plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Congrats! You did it! :)"
      ],
      "metadata": {
        "id": "eoWDEy6CwDr6"
      }
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "gpuClass": "premium",
      "gpuType": "T4"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.13"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}