07_02_lstm_compose_train.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compose: Training a model to generate music"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/davidfoster/.virtualenvs/gdl/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pickle\n",
    "import numpy\n",
    "from music21 import note, chord\n",
    "\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "from keras.utils import plot_model\n",
    "\n",
    "from models.RNNAttention import get_distinct, create_lookups, prepare_sequences, get_music_list, create_network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run params\n",
    "section = 'compose'\n",
    "run_id = '0006'\n",
    "music_name = 'cello'\n",
    "\n",
    "run_folder = 'run/{}/'.format(section)\n",
    "run_folder += '_'.join([run_id, music_name])\n",
    "\n",
    "\n",
    "store_folder = os.path.join(run_folder, 'store')\n",
    "data_folder = os.path.join('data', music_name)\n",
    "\n",
    "if not os.path.exists(run_folder):\n",
    "    os.mkdir(run_folder)\n",
    "    os.mkdir(os.path.join(run_folder, 'store'))\n",
    "    os.mkdir(os.path.join(run_folder, 'output'))\n",
    "    os.mkdir(os.path.join(run_folder, 'weights'))\n",
    "    os.mkdir(os.path.join(run_folder, 'viz'))\n",
    "    \n",
    "\n",
    "\n",
    "mode = 'build' # 'load' # \n",
    "\n",
    "# data params\n",
    "intervals = range(1)\n",
    "seq_len = 32\n",
    "\n",
    "# model params\n",
    "embed_size = 100\n",
    "rnn_units = 256\n",
    "use_attention = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract the notes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35 files in total\n",
      "1 Parsing data/cello/cs1-2all.mid\n",
      "2 Parsing data/cello/cs5-1pre.mid\n",
      "3 Parsing data/cello/cs4-1pre.mid\n",
      "4 Parsing data/cello/cs3-5bou.mid\n",
      "5 Parsing data/cello/cs1-4sar.mid\n",
      "6 Parsing data/cello/cs2-5men.mid\n",
      "7 Parsing data/cello/cs3-3cou.mid\n",
      "8 Parsing data/cello/cs2-3cou.mid\n",
      "9 Parsing data/cello/cs1-6gig.mid\n",
      "10 Parsing data/cello/cs6-4sar.mid\n",
      "11 Parsing data/cello/cs4-5bou.mid\n",
      "12 Parsing data/cello/cs4-3cou.mid\n",
      "13 Parsing data/cello/cs5-3cou.mid\n",
      "14 Parsing data/cello/cs6-5gav.mid\n",
      "15 Parsing data/cello/cs6-6gig.mid\n",
      "16 Parsing data/cello/cs2-1pre.mid\n",
      "17 Parsing data/cello/cs3-1pre.mid\n",
      "18 Parsing data/cello/cs3-6gig.mid\n",
      "19 Parsing data/cello/cs2-6gig.mid\n",
      "20 Parsing data/cello/cs2-4sar.mid\n",
      "21 Parsing data/cello/cs3-4sar.mid\n",
      "22 Parsing data/cello/cs1-5men.mid\n",
      "23 Parsing data/cello/cs1-3cou.mid\n",
      "24 Parsing data/cello/cs6-1pre.mid\n",
      "25 Parsing data/cello/cs2-2all.mid\n",
      "26 Parsing data/cello/cs3-2all.mid\n",
      "27 Parsing data/cello/cs1-1pre.mid\n",
      "28 Parsing data/cello/cs5-2all.mid\n",
      "29 Parsing data/cello/cs4-2all.mid\n",
      "30 Parsing data/cello/cs5-5gav.mid\n",
      "31 Parsing data/cello/cs4-6gig.mid\n",
      "32 Parsing data/cello/cs5-6gig.mid\n",
      "33 Parsing data/cello/cs5-4sar.mid\n",
      "34 Parsing data/cello/cs4-4sar.mid\n",
      "35 Parsing data/cello/cs6-3cou.mid\n"
     ]
    }
   ],
   "source": [
    "if mode == 'build':\n",
    "    \n",
    "    music_list, parser = get_music_list(data_folder)\n",
    "    print(len(music_list), 'files in total')\n",
    "\n",
    "    notes = []\n",
    "    durations = []\n",
    "\n",
    "    for i, file in enumerate(music_list):\n",
    "        print(i+1, \"Parsing %s\" % file)\n",
    "        original_score = parser.parse(file).chordify()\n",
    "        \n",
    "\n",
    "        for interval in intervals:\n",
    "\n",
    "            score = original_score.transpose(interval)\n",
    "\n",
    "            notes.extend(['START'] * seq_len)\n",
    "            durations.extend([0]* seq_len)\n",
    "\n",
    "            for element in score.flat:\n",
    "                \n",
    "                if isinstance(element, note.Note):\n",
    "                    if element.isRest:\n",
    "                        notes.append(str(element.name))\n",
    "                        durations.append(element.duration.quarterLength)\n",
    "                    else:\n",
    "                        notes.append(str(element.nameWithOctave))\n",
    "                        durations.append(element.duration.quarterLength)\n",
    "\n",
    "                if isinstance(element, chord.Chord):\n",
    "                    notes.append('.'.join(n.nameWithOctave for n in element.pitches))\n",
    "                    durations.append(element.duration.quarterLength)\n",
    "\n",
    "    with open(os.path.join(store_folder, 'notes'), 'wb') as f:\n",
    "        pickle.dump(notes, f) #['G2', 'D3', 'B3', 'A3', 'B3', 'D3', 'B3', 'D3', 'G2',...]\n",
    "    with open(os.path.join(store_folder, 'durations'), 'wb') as f:\n",
    "        pickle.dump(durations, f) \n",
    "else:\n",
    "    with open(os.path.join(store_folder, 'notes'), 'rb') as f:\n",
    "        notes = pickle.load(f) #['G2', 'D3', 'B3', 'A3', 'B3', 'D3', 'B3', 'D3', 'G2',...]\n",
    "    with open(os.path.join(store_folder, 'durations'), 'rb') as f:\n",
    "        durations = pickle.load(f) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create the lookup tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the distinct sets of notes and durations\n",
    "note_names, n_notes = get_distinct(notes)\n",
    "duration_names, n_durations = get_distinct(durations)\n",
    "distincts = [note_names, n_notes, duration_names, n_durations]\n",
    "\n",
    "with open(os.path.join(store_folder, 'distincts'), 'wb') as f:\n",
    "    pickle.dump(distincts, f)\n",
    "\n",
    "# make the lookup dictionaries for notes and dictionaries and save\n",
    "note_to_int, int_to_note = create_lookups(note_names)\n",
    "duration_to_int, int_to_duration = create_lookups(duration_names)\n",
    "lookups = [note_to_int, int_to_note, duration_to_int, int_to_duration]\n",
    "\n",
    "with open(os.path.join(store_folder, 'lookups'), 'wb') as f:\n",
    "    pickle.dump(lookups, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "note_to_int\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'A2': 0,\n",
       " 'A2.A3': 1,\n",
       " 'A2.B2': 2,\n",
       " 'A2.C3': 3,\n",
       " 'A2.D3': 4,\n",
       " 'A2.E-3': 5,\n",
       " 'A2.E3': 6,\n",
       " 'A2.E3.A3': 7,\n",
       " 'A2.E3.C#4': 8,\n",
       " 'A2.E3.C#4.A4': 9,\n",
       " 'A2.E3.C#4.E4': 10,\n",
       " 'A2.E3.C#4.G#4': 11,\n",
       " 'A2.E3.C4': 12,\n",
       " 'A2.E3.D4': 13,\n",
       " 'A2.F#3': 14,\n",
       " 'A2.F#3.C4': 15,\n",
       " 'A2.F#3.D4': 16,\n",
       " 'A2.F#3.D4.A4': 17,\n",
       " 'A2.F#3.D4.E4': 18,\n",
       " 'A2.F#3.D4.F#4': 19,\n",
       " 'A2.F3': 20,\n",
       " 'A2.F3.C4': 21,\n",
       " 'A2.F3.D4': 22,\n",
       " 'A2.F3.D4.A4': 23,\n",
       " 'A2.G3': 24,\n",
       " 'A2.G3.C#4': 25,\n",
       " 'A2.G3.D4': 26,\n",
       " 'A3': 27,\n",
       " 'A3.B-3': 28,\n",
       " 'A3.B3': 29,\n",
       " 'A3.C#4.E4': 30,\n",
       " 'A3.C4': 31,\n",
       " 'A3.D4': 32,\n",
       " 'A3.E4': 33,\n",
       " 'A3.E4.F#4': 34,\n",
       " 'A3.E4.G4': 35,\n",
       " 'A3.F#4': 36,\n",
       " 'A3.F4': 37,\n",
       " 'A3.G4': 38,\n",
       " 'A4': 39,\n",
       " 'B-2': 40,\n",
       " 'B-2.A3': 41,\n",
       " 'B-2.B-3': 42,\n",
       " 'B-2.D3': 43,\n",
       " 'B-2.D3.A3': 44,\n",
       " 'B-2.D3.E-3.G#3': 45,\n",
       " 'B-2.D3.G#3': 46,\n",
       " 'B-2.E-3': 47,\n",
       " 'B-2.E3': 48,\n",
       " 'B-2.E3.D4': 49,\n",
       " 'B-2.F#3.C#4.E4': 50,\n",
       " 'B-2.F3': 51,\n",
       " 'B-2.F3.C4': 52,\n",
       " 'B-2.F3.D4': 53,\n",
       " 'B-2.F3.E-4': 54,\n",
       " 'B-2.G#3': 55,\n",
       " 'B-2.G3': 56,\n",
       " 'B-2.G3.D4': 57,\n",
       " 'B-3': 58,\n",
       " 'B-3.C4': 59,\n",
       " 'B-4': 60,\n",
       " 'B2': 61,\n",
       " 'B2.A3': 62,\n",
       " 'B2.C3': 63,\n",
       " 'B2.D3': 64,\n",
       " 'B2.D3.B3.F#4': 65,\n",
       " 'B2.E-3': 66,\n",
       " 'B2.E3.D4': 67,\n",
       " 'B2.E3.E4': 68,\n",
       " 'B2.F#3': 69,\n",
       " 'B2.F#3.B3': 70,\n",
       " 'B2.F#3.D4': 71,\n",
       " 'B2.F#3.E-4.A4': 72,\n",
       " 'B2.F3': 73,\n",
       " 'B2.G#3': 74,\n",
       " 'B2.G#3.D4': 75,\n",
       " 'B2.G3': 76,\n",
       " 'B3': 77,\n",
       " 'B3.C4': 78,\n",
       " 'B3.D4': 79,\n",
       " 'B3.E4': 80,\n",
       " 'B3.F#4': 81,\n",
       " 'B3.G4': 82,\n",
       " 'B4': 83,\n",
       " 'C#2': 84,\n",
       " 'C#2.B-2.F3': 85,\n",
       " 'C#2.B-2.G3': 86,\n",
       " 'C#3': 87,\n",
       " 'C#3.D3': 88,\n",
       " 'C#3.E3.A3': 89,\n",
       " 'C#3.E3.B3.E4': 90,\n",
       " 'C#3.G3': 91,\n",
       " 'C#3.G3.A3': 92,\n",
       " 'C#3.G3.B-3': 93,\n",
       " 'C#4': 94,\n",
       " 'C#4.A4': 95,\n",
       " 'C#4.D4': 96,\n",
       " 'C#4.E4': 97,\n",
       " 'C#5': 98,\n",
       " 'C2': 99,\n",
       " 'C2.A2.E-3': 100,\n",
       " 'C2.A2.F#3': 101,\n",
       " 'C2.A2.F#3.D4': 102,\n",
       " 'C2.A2.F3.A3': 103,\n",
       " 'C2.A2.G3.A3': 104,\n",
       " 'C2.B-2': 105,\n",
       " 'C2.B-2.E3': 106,\n",
       " 'C2.B2.F3.G#3': 107,\n",
       " 'C2.C3': 108,\n",
       " 'C2.E-3': 109,\n",
       " 'C2.E-3.G3': 110,\n",
       " 'C2.E3.G3': 111,\n",
       " 'C2.G#2': 112,\n",
       " 'C2.G#2.D3.B3': 113,\n",
       " 'C2.G2': 114,\n",
       " 'C2.G2.D3': 115,\n",
       " 'C2.G2.E-3': 116,\n",
       " 'C2.G2.E-3.B-3': 117,\n",
       " 'C2.G2.E-3.C4': 118,\n",
       " 'C2.G2.E3': 119,\n",
       " 'C2.G2.E3.B-3': 120,\n",
       " 'C2.G2.E3.C4': 121,\n",
       " 'C2.G2.F3': 122,\n",
       " 'C2.G2.F3.C4': 123,\n",
       " 'C3': 124,\n",
       " 'C3.A3.E-4': 125,\n",
       " 'C3.B-3': 126,\n",
       " 'C3.D3': 127,\n",
       " 'C3.E-3': 128,\n",
       " 'C3.E3': 129,\n",
       " 'C3.E3.A3': 130,\n",
       " 'C3.E3.B-3': 131,\n",
       " 'C3.E3.B3': 132,\n",
       " 'C3.E3.E4': 133,\n",
       " 'C3.E3.F#4': 134,\n",
       " 'C3.F#3': 135,\n",
       " 'C3.F3': 136,\n",
       " 'C3.G#3': 137,\n",
       " 'C3.G#3.E-4': 138,\n",
       " 'C3.G3': 139,\n",
       " 'C3.G3.A3': 140,\n",
       " 'C3.G3.B-3': 141,\n",
       " 'C3.G3.C4': 142,\n",
       " 'C3.G3.E-4': 143,\n",
       " 'C4': 144,\n",
       " 'C4.D4': 145,\n",
       " 'C4.E4': 146,\n",
       " 'C4.F#4': 147,\n",
       " 'C5': 148,\n",
       " 'D2': 149,\n",
       " 'D2.A2.D3': 150,\n",
       " 'D2.A2.F#3': 151,\n",
       " 'D2.A2.F#3.C#4': 152,\n",
       " 'D2.A2.F#3.C4': 153,\n",
       " 'D2.A2.F#3.D4': 154,\n",
       " 'D2.A2.F3': 155,\n",
       " 'D2.A2.F3.A3': 156,\n",
       " 'D2.A2.F3.D4': 157,\n",
       " 'D2.B-2.F3.G#3': 158,\n",
       " 'D2.B-2.G#3': 159,\n",
       " 'D2.B-2.G3.C4': 160,\n",
       " 'D2.B-2.G3.D4': 161,\n",
       " 'D2.B2': 162,\n",
       " 'D2.B2.F3': 163,\n",
       " 'D2.B2.G3': 164,\n",
       " 'D2.C3.F#3.E-4': 165,\n",
       " 'D2.E-3': 166,\n",
       " 'D2.G2.F3.B3': 167,\n",
       " 'D3': 168,\n",
       " 'D3.A3': 169,\n",
       " 'D3.A3.F#4': 170,\n",
       " 'D3.A3.G4': 171,\n",
       " 'D3.A4': 172,\n",
       " 'D3.B-3': 173,\n",
       " 'D3.B3': 174,\n",
       " 'D3.B3.G4': 175,\n",
       " 'D3.C#4': 176,\n",
       " 'D3.C#4.E4': 177,\n",
       " 'D3.C#4.F#4': 178,\n",
       " 'D3.C4': 179,\n",
       " 'D3.C4.F#4': 180,\n",
       " 'D3.D4': 181,\n",
       " 'D3.E-3': 182,\n",
       " 'D3.E-4': 183,\n",
       " 'D3.E3': 184,\n",
       " 'D3.E4': 185,\n",
       " 'D3.F#3': 186,\n",
       " 'D3.F#4': 187,\n",
       " 'D3.F3': 188,\n",
       " 'D3.F3.A3': 189,\n",
       " 'D3.F4': 190,\n",
       " 'D3.G#3': 191,\n",
       " 'D3.G3': 192,\n",
       " 'D3.G3.A3': 193,\n",
       " 'D3.G3.G#3': 194,\n",
       " 'D3.G4': 195,\n",
       " 'D4': 196,\n",
       " 'D4.A4': 197,\n",
       " 'D4.B4': 198,\n",
       " 'D4.C5': 199,\n",
       " 'D4.E-4': 200,\n",
       " 'D4.E4': 201,\n",
       " 'D4.G#4': 202,\n",
       " 'D5': 203,\n",
       " 'E-2': 204,\n",
       " 'E-2.B-2.E-3': 205,\n",
       " 'E-2.B-2.E-3.G3': 206,\n",
       " 'E-2.B-2.F3': 207,\n",
       " 'E-2.B-2.G#3': 208,\n",
       " 'E-2.B-2.G3': 209,\n",
       " 'E-2.B-2.G3.E-4': 210,\n",
       " 'E-2.C3': 211,\n",
       " 'E-2.G2': 212,\n",
       " 'E-2.G2.D3': 213,\n",
       " 'E-2.G2.G3.A3': 214,\n",
       " 'E-2.G3': 215,\n",
       " 'E-3': 216,\n",
       " 'E-3.A3': 217,\n",
       " 'E-3.B-3': 218,\n",
       " 'E-3.C#4': 219,\n",
       " 'E-3.C4': 220,\n",
       " 'E-3.C4.F#4': 221,\n",
       " 'E-3.D4': 222,\n",
       " 'E-3.F3': 223,\n",
       " 'E-3.F3.G3': 224,\n",
       " 'E-3.G#3': 225,\n",
       " 'E-3.G3': 226,\n",
       " 'E-4': 227,\n",
       " 'E-4.F4': 228,\n",
       " 'E-5': 229,\n",
       " 'E2': 230,\n",
       " 'E2.B2.G#3.D4': 231,\n",
       " 'E2.B2.G3': 232,\n",
       " 'E2.C3.G3': 233,\n",
       " 'E2.C3.G3.B-3': 234,\n",
       " 'E2.E3.G3': 235,\n",
       " 'E2.G2': 236,\n",
       " 'E3': 237,\n",
       " 'E3.A3': 238,\n",
       " 'E3.B-3': 239,\n",
       " 'E3.B3': 240,\n",
       " 'E3.B3.A4': 241,\n",
       " 'E3.B3.F#4': 242,\n",
       " 'E3.B3.G4': 243,\n",
       " 'E3.C#4': 244,\n",
       " 'E3.C#4.G4': 245,\n",
       " 'E3.C4': 246,\n",
       " 'E3.D4': 247,\n",
       " 'E3.D4.G#4': 248,\n",
       " 'E3.F#3': 249,\n",
       " 'E3.F3': 250,\n",
       " 'E3.G3': 251,\n",
       " 'E4': 252,\n",
       " 'E4.G4': 253,\n",
       " 'E5': 254,\n",
       " 'F#2': 255,\n",
       " 'F#2.A2.D3.A3': 256,\n",
       " 'F#2.C3.E-3.C4': 257,\n",
       " 'F#2.E3.C#4.E4': 258,\n",
       " 'F#2.G2': 259,\n",
       " 'F#3': 260,\n",
       " 'F#3.A3': 261,\n",
       " 'F#3.A3.E4': 262,\n",
       " 'F#3.B3': 263,\n",
       " 'F#3.B3.A4': 264,\n",
       " 'F#3.C#4': 265,\n",
       " 'F#3.C#4.A4': 266,\n",
       " 'F#3.C4': 267,\n",
       " 'F#3.D4': 268,\n",
       " 'F#3.D4.A4': 269,\n",
       " 'F#3.G#3': 270,\n",
       " 'F#3.G3': 271,\n",
       " 'F#4': 272,\n",
       " 'F#4.A4': 273,\n",
       " 'F#4.D5': 274,\n",
       " 'F#4.G4': 275,\n",
       " 'F#5': 276,\n",
       " 'F2': 277,\n",
       " 'F2.A2': 278,\n",
       " 'F2.A2.D3': 279,\n",
       " 'F2.A2.D3.A3': 280,\n",
       " 'F2.A2.E3': 281,\n",
       " 'F2.A3': 282,\n",
       " 'F2.B-2': 283,\n",
       " 'F2.B2.G3': 284,\n",
       " 'F2.C3': 285,\n",
       " 'F2.C3.D3': 286,\n",
       " 'F2.C3.E-3': 287,\n",
       " 'F2.C3.G#3': 288,\n",
       " 'F2.D3.G#3': 289,\n",
       " 'F2.G#2.D3': 290,\n",
       " 'F2.G#2.E-3': 291,\n",
       " 'F2.G2.D3.B3': 292,\n",
       " 'F3': 293,\n",
       " 'F3.A3': 294,\n",
       " 'F3.B-3': 295,\n",
       " 'F3.B3': 296,\n",
       " 'F3.C4': 297,\n",
       " 'F3.D4': 298,\n",
       " 'F3.E4': 299,\n",
       " 'F3.G#3': 300,\n",
       " 'F3.G3': 301,\n",
       " 'F4': 302,\n",
       " 'G#2': 303,\n",
       " 'G#2.B-2': 304,\n",
       " 'G#2.D3': 305,\n",
       " 'G#2.D3.B3': 306,\n",
       " 'G#2.E-3': 307,\n",
       " 'G#2.E-3.B-3': 308,\n",
       " 'G#2.E-3.C#4': 309,\n",
       " 'G#2.E-3.C4': 310,\n",
       " 'G#2.E3.B3': 311,\n",
       " 'G#2.E3.C#4': 312,\n",
       " 'G#2.E3.D4': 313,\n",
       " 'G#2.E3.D4.B4': 314,\n",
       " 'G#2.F3': 315,\n",
       " 'G#2.F3.B3': 316,\n",
       " 'G#2.F3.C4': 317,\n",
       " 'G#2.F3.D4': 318,\n",
       " 'G#2.F3.E-4': 319,\n",
       " 'G#2.G3': 320,\n",
       " 'G#3': 321,\n",
       " 'G#3.A3': 322,\n",
       " 'G#3.B-3': 323,\n",
       " 'G#3.D4': 324,\n",
       " 'G#3.D4.E4': 325,\n",
       " 'G#3.F4': 326,\n",
       " 'G#4': 327,\n",
       " 'G2': 328,\n",
       " 'G2.A3': 329,\n",
       " 'G2.B-3': 330,\n",
       " 'G2.B2': 331,\n",
       " 'G2.B3': 332,\n",
       " 'G2.B3.D4': 333,\n",
       " 'G2.B3.G4': 334,\n",
       " 'G2.C4': 335,\n",
       " 'G2.D3': 336,\n",
       " 'G2.D3.A3': 337,\n",
       " 'G2.D3.A3.G4': 338,\n",
       " 'G2.D3.B-3': 339,\n",
       " 'G2.D3.B3': 340,\n",
       " 'G2.D3.B3.A4': 341,\n",
       " 'G2.D3.B3.F#4': 342,\n",
       " 'G2.D3.C#4': 343,\n",
       " 'G2.D3.C4': 344,\n",
       " 'G2.D3.C4.G4': 345,\n",
       " 'G2.D3.D4': 346,\n",
       " 'G2.D3.G3': 347,\n",
       " 'G2.E-3': 348,\n",
       " 'G2.E-3.B-3': 349,\n",
       " 'G2.E3': 350,\n",
       " 'G2.E3.B3': 351,\n",
       " 'G2.E3.C#4': 352,\n",
       " 'G2.E3.C#4.A4': 353,\n",
       " 'G2.E3.C4': 354,\n",
       " 'G2.E3.D4': 355,\n",
       " 'G2.E3.F3': 356,\n",
       " 'G2.F#3': 357,\n",
       " 'G2.F#3.D4': 358,\n",
       " 'G2.F#3.E-4': 359,\n",
       " 'G2.F3': 360,\n",
       " 'G2.F3.B-3': 361,\n",
       " 'G2.F3.B3': 362,\n",
       " 'G2.F3.C4': 363,\n",
       " 'G2.F3.D4': 364,\n",
       " 'G2.F3.G3': 365,\n",
       " 'G2.G#3': 366,\n",
       " 'G2.G3': 367,\n",
       " 'G2.G3.B3.C#4': 368,\n",
       " 'G2.G3.B3.E4': 369,\n",
       " 'G2.G3.E4': 370,\n",
       " 'G3': 371,\n",
       " 'G3.A3': 372,\n",
       " 'G3.B-3': 373,\n",
       " 'G3.B3': 374,\n",
       " 'G3.B3.B4': 375,\n",
       " 'G3.B3.E4': 376,\n",
       " 'G3.B3.F#4': 377,\n",
       " 'G3.C#4': 378,\n",
       " 'G3.C4': 379,\n",
       " 'G3.D4': 380,\n",
       " 'G3.D4.B4': 381,\n",
       " 'G3.E-4': 382,\n",
       " 'G3.E4': 383,\n",
       " 'G4': 384,\n",
       " 'G5': 385,\n",
       " 'START': 386}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('\\nnote_to_int')\n",
    "note_to_int"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "duration_to_int\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{0: 0,\n",
       " Fraction(1, 12): 1,\n",
       " Fraction(1, 6): 2,\n",
       " 0.25: 3,\n",
       " Fraction(1, 3): 4,\n",
       " 0.5: 5,\n",
       " Fraction(2, 3): 6,\n",
       " 0.75: 7,\n",
       " 1.0: 8,\n",
       " 1.25: 9,\n",
       " Fraction(4, 3): 10,\n",
       " 1.5: 11,\n",
       " 1.75: 12,\n",
       " 2.0: 13,\n",
       " 2.25: 14,\n",
       " 2.5: 15,\n",
       " 3.0: 16,\n",
       " 4.0: 17}"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('\\nduration_to_int')\n",
    "duration_to_int"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare the sequences used by the Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "network_input, network_output = prepare_sequences(notes, durations, lookups, distincts, seq_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pitch input\n",
      "[386 386 386 386 386 386 386 386 386 386 386 386 386 386 386 386 386 386\n",
      " 386 386 386 386 386 386 386 386 386 386 386 386 386 386]\n",
      "duration input\n",
      "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      "pitch output\n",
      "[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0.]\n",
      "duration output\n",
      "[0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n"
     ]
    }
   ],
   "source": [
    "print('pitch input')\n",
    "print(network_input[0][0])\n",
    "print('duration input')\n",
    "print(network_input[1][0])\n",
    "print('pitch output')\n",
    "print(network_output[0][0])\n",
    "print('duration output')\n",
    "print(network_output[1][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create the structure of the neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, None, 100)    38700       input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, None, 100)    1800        input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, None, 200)    0           embedding_1[0][0]                \n",
      "                                                                 embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   (None, None, 256)    467968      concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lstm_2 (LSTM)                   (None, None, 256)    525312      lstm_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, None, 1)      257         lstm_2[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "reshape_1 (Reshape)             (None, None)         0           dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, None)         0           reshape_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "repeat_vector_1 (RepeatVector)  (None, 256, None)    0           activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "permute_1 (Permute)             (None, None, 256)    0           repeat_vector_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "multiply_1 (Multiply)           (None, None, 256)    0           lstm_2[0][0]                     \n",
      "                                                                 permute_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)               (None, 256)          0           multiply_1[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "pitch (Dense)                   (None, 387)          99459       lambda_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "duration (Dense)                (None, 18)           4626        lambda_1[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 1,138,122\n",
      "Trainable params: 1,138,122\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model, att_model = create_network(n_notes, n_durations, embed_size, rnn_units, use_attention)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_model(model, to_file=os.path.join(run_folder ,'viz/model.png'), show_shapes = True, show_layer_names = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights_folder = os.path.join(run_folder, 'weights')\n",
    "# model.load_weights(os.path.join(weights_folder, \"weights.h5\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 22376 samples, validate on 5595 samples\n",
      "Epoch 1/2000000\n",
      "22376/22376 [==============================] - 107s 5ms/step - loss: 4.2511 - pitch_loss: 3.4720 - duration_loss: 0.7791 - val_loss: 3.9800 - val_pitch_loss: 3.2307 - val_duration_loss: 0.7493\n",
      "Epoch 2/2000000\n",
      "22376/22376 [==============================] - 101s 4ms/step - loss: 3.7509 - pitch_loss: 3.1665 - duration_loss: 0.5845 - val_loss: 3.8029 - val_pitch_loss: 3.1389 - val_duration_loss: 0.6640\n",
      "Epoch 3/2000000\n",
      "22376/22376 [==============================] - 104s 5ms/step - loss: 3.5442 - pitch_loss: 3.0222 - duration_loss: 0.5220 - val_loss: 3.7914 - val_pitch_loss: 3.0135 - val_duration_loss: 0.7779\n",
      "Epoch 4/2000000\n",
      "22376/22376 [==============================] - 100s 4ms/step - loss: 3.3763 - pitch_loss: 2.8962 - duration_loss: 0.4801 - val_loss: 3.6327 - val_pitch_loss: 2.9615 - val_duration_loss: 0.6712\n",
      "Epoch 5/2000000\n",
      "22376/22376 [==============================] - 101s 5ms/step - loss: 3.2341 - pitch_loss: 2.7942 - duration_loss: 0.4400 - val_loss: 3.6039 - val_pitch_loss: 2.9722 - val_duration_loss: 0.6317\n",
      "Epoch 6/2000000\n",
      "22376/22376 [==============================] - 99s 4ms/step - loss: 3.0845 - pitch_loss: 2.6858 - duration_loss: 0.3987 - val_loss: 3.6359 - val_pitch_loss: 3.0142 - val_duration_loss: 0.6216\n",
      "Epoch 7/2000000\n",
      "22376/22376 [==============================] - 102s 5ms/step - loss: 2.9448 - pitch_loss: 2.5818 - duration_loss: 0.3630 - val_loss: 3.8273 - val_pitch_loss: 3.0810 - val_duration_loss: 0.7463\n",
      "Epoch 8/2000000\n",
      "22376/22376 [==============================] - 101s 4ms/step - loss: 2.8101 - pitch_loss: 2.4827 - duration_loss: 0.3275 - val_loss: 3.7175 - val_pitch_loss: 3.0621 - val_duration_loss: 0.6554\n",
      "Epoch 9/2000000\n",
      "22376/22376 [==============================] - 100s 4ms/step - loss: 2.6715 - pitch_loss: 2.3766 - duration_loss: 0.2949 - val_loss: 3.8268 - val_pitch_loss: 3.0995 - val_duration_loss: 0.7273\n",
      "Epoch 10/2000000\n",
      "22376/22376 [==============================] - 102s 5ms/step - loss: 2.5359 - pitch_loss: 2.2704 - duration_loss: 0.2655 - val_loss: 3.8912 - val_pitch_loss: 3.1205 - val_duration_loss: 0.7708\n",
      "Epoch 11/2000000\n",
      "22376/22376 [==============================] - 105s 5ms/step - loss: 2.3940 - pitch_loss: 2.1594 - duration_loss: 0.2346 - val_loss: 3.9222 - val_pitch_loss: 3.2240 - val_duration_loss: 0.6982\n",
      "Epoch 12/2000000\n",
      "22376/22376 [==============================] - 109s 5ms/step - loss: 2.2525 - pitch_loss: 2.0424 - duration_loss: 0.2102 - val_loss: 4.0419 - val_pitch_loss: 3.2651 - val_duration_loss: 0.7768\n",
      "Epoch 13/2000000\n",
      "22376/22376 [==============================] - 107s 5ms/step - loss: 2.1127 - pitch_loss: 1.9259 - duration_loss: 0.1868 - val_loss: 4.0901 - val_pitch_loss: 3.3233 - val_duration_loss: 0.7668\n",
      "Epoch 14/2000000\n",
      "22376/22376 [==============================] - 108s 5ms/step - loss: 1.9808 - pitch_loss: 1.8135 - duration_loss: 0.1673 - val_loss: 4.2289 - val_pitch_loss: 3.3936 - val_duration_loss: 0.8354\n",
      "Epoch 15/2000000\n",
      "22376/22376 [==============================] - 103s 5ms/step - loss: 1.8396 - pitch_loss: 1.6941 - duration_loss: 0.1455 - val_loss: 4.3158 - val_pitch_loss: 3.4862 - val_duration_loss: 0.8297\n",
      "Epoch 16/2000000\n",
      "22376/22376 [==============================] - 108s 5ms/step - loss: 1.7201 - pitch_loss: 1.5891 - duration_loss: 0.1309 - val_loss: 4.5088 - val_pitch_loss: 3.6002 - val_duration_loss: 0.9086\n",
      "Epoch 17/2000000\n",
      "22376/22376 [==============================] - 106s 5ms/step - loss: 1.6055 - pitch_loss: 1.4897 - duration_loss: 0.1158 - val_loss: 4.6276 - val_pitch_loss: 3.7062 - val_duration_loss: 0.9214\n",
      "Epoch 18/2000000\n",
      "22376/22376 [==============================] - 104s 5ms/step - loss: 1.5004 - pitch_loss: 1.3953 - duration_loss: 0.1051 - val_loss: 4.7970 - val_pitch_loss: 3.8080 - val_duration_loss: 0.9890\n",
      "Epoch 19/2000000\n",
      "22376/22376 [==============================] - 106s 5ms/step - loss: 1.4084 - pitch_loss: 1.3104 - duration_loss: 0.0979 - val_loss: 4.9941 - val_pitch_loss: 3.9899 - val_duration_loss: 1.0042\n",
      "Epoch 20/2000000\n",
      "22376/22376 [==============================] - 107s 5ms/step - loss: 1.3164 - pitch_loss: 1.2289 - duration_loss: 0.0875 - val_loss: 5.1327 - val_pitch_loss: 4.1694 - val_duration_loss: 0.9633\n",
      "Epoch 21/2000000\n",
      "22376/22376 [==============================] - 101s 4ms/step - loss: 1.2372 - pitch_loss: 1.1557 - duration_loss: 0.0814 - val_loss: 5.1450 - val_pitch_loss: 4.0532 - val_duration_loss: 1.0918\n",
      "Epoch 22/2000000\n",
      "22376/22376 [==============================] - 100s 4ms/step - loss: 1.1640 - pitch_loss: 1.0905 - duration_loss: 0.0735 - val_loss: 5.2690 - val_pitch_loss: 4.2024 - val_duration_loss: 1.0667\n",
      "Epoch 23/2000000\n",
      "22376/22376 [==============================] - 102s 5ms/step - loss: 1.0997 - pitch_loss: 1.0286 - duration_loss: 0.0710 - val_loss: 5.4709 - val_pitch_loss: 4.3984 - val_duration_loss: 1.0725\n",
      "Epoch 24/2000000\n",
      "22376/22376 [==============================] - 99s 4ms/step - loss: 1.0452 - pitch_loss: 0.9778 - duration_loss: 0.0675 - val_loss: 5.4866 - val_pitch_loss: 4.3843 - val_duration_loss: 1.1023\n",
      "Epoch 25/2000000\n",
      "22376/22376 [==============================] - 102s 5ms/step - loss: 0.9851 - pitch_loss: 0.9240 - duration_loss: 0.0611 - val_loss: 5.5974 - val_pitch_loss: 4.4493 - val_duration_loss: 1.1481\n",
      "Epoch 26/2000000\n",
      "22376/22376 [==============================] - 102s 5ms/step - loss: 0.9427 - pitch_loss: 0.8799 - duration_loss: 0.0629 - val_loss: 5.6729 - val_pitch_loss: 4.6150 - val_duration_loss: 1.0579\n",
      "Epoch 27/2000000\n",
      "22376/22376 [==============================] - 100s 4ms/step - loss: 0.8909 - pitch_loss: 0.8310 - duration_loss: 0.0599 - val_loss: 5.7451 - val_pitch_loss: 4.6365 - val_duration_loss: 1.1085\n",
      "Epoch 28/2000000\n",
      "22376/22376 [==============================] - 102s 5ms/step - loss: 0.8426 - pitch_loss: 0.7905 - duration_loss: 0.0521 - val_loss: 6.0188 - val_pitch_loss: 4.8628 - val_duration_loss: 1.1561\n",
      "Epoch 29/2000000\n",
      "22376/22376 [==============================] - 102s 5ms/step - loss: 0.8121 - pitch_loss: 0.7581 - duration_loss: 0.0541 - val_loss: 6.1081 - val_pitch_loss: 4.8904 - val_duration_loss: 1.2177\n",
      "Epoch 30/2000000\n",
      "22376/22376 [==============================] - 100s 4ms/step - loss: 0.7713 - pitch_loss: 0.7228 - duration_loss: 0.0486 - val_loss: 6.1590 - val_pitch_loss: 4.9724 - val_duration_loss: 1.1866\n",
      "Epoch 31/2000000\n",
      "22376/22376 [==============================] - 100s 4ms/step - loss: 0.7475 - pitch_loss: 0.6970 - duration_loss: 0.0505 - val_loss: 6.2072 - val_pitch_loss: 5.0141 - val_duration_loss: 1.1931\n",
      "Epoch 32/2000000\n",
      "22376/22376 [==============================] - 105s 5ms/step - loss: 0.7153 - pitch_loss: 0.6689 - duration_loss: 0.0464 - val_loss: 6.1217 - val_pitch_loss: 4.9305 - val_duration_loss: 1.1912\n",
      "Epoch 33/2000000\n",
      "22376/22376 [==============================] - 110s 5ms/step - loss: 0.6832 - pitch_loss: 0.6371 - duration_loss: 0.0461 - val_loss: 6.3682 - val_pitch_loss: 5.1281 - val_duration_loss: 1.2400\n",
      "Epoch 34/2000000\n",
      "22376/22376 [==============================] - 105s 5ms/step - loss: 0.6555 - pitch_loss: 0.6117 - duration_loss: 0.0438 - val_loss: 6.2355 - val_pitch_loss: 5.0259 - val_duration_loss: 1.2095\n",
      "Epoch 35/2000000\n",
      "22376/22376 [==============================] - 105s 5ms/step - loss: 0.6300 - pitch_loss: 0.5881 - duration_loss: 0.0418 - val_loss: 6.4605 - val_pitch_loss: 5.2335 - val_duration_loss: 1.2271\n",
      "Epoch 36/2000000\n",
      "22376/22376 [==============================] - 107s 5ms/step - loss: 0.6086 - pitch_loss: 0.5683 - duration_loss: 0.0403 - val_loss: 6.2863 - val_pitch_loss: 5.0709 - val_duration_loss: 1.2154\n",
      "Epoch 37/2000000\n",
      "22376/22376 [==============================] - 109s 5ms/step - loss: 0.5907 - pitch_loss: 0.5506 - duration_loss: 0.0401 - val_loss: 6.4853 - val_pitch_loss: 5.1744 - val_duration_loss: 1.3109\n",
      "Epoch 38/2000000\n",
      "22376/22376 [==============================] - 102s 5ms/step - loss: 0.5733 - pitch_loss: 0.5347 - duration_loss: 0.0386 - val_loss: 6.6464 - val_pitch_loss: 5.3507 - val_duration_loss: 1.2956\n",
      "Epoch 39/2000000\n",
      "22376/22376 [==============================] - 107s 5ms/step - loss: 0.5594 - pitch_loss: 0.5199 - duration_loss: 0.0396 - val_loss: 6.4648 - val_pitch_loss: 5.2441 - val_duration_loss: 1.2207\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 40/2000000\n",
      "22376/22376 [==============================] - 101s 5ms/step - loss: 0.5359 - pitch_loss: 0.4968 - duration_loss: 0.0391 - val_loss: 6.5611 - val_pitch_loss: 5.3147 - val_duration_loss: 1.2464\n",
      "Epoch 41/2000000\n",
      "22376/22376 [==============================] - 100s 4ms/step - loss: 0.5283 - pitch_loss: 0.4917 - duration_loss: 0.0366 - val_loss: 6.4475 - val_pitch_loss: 5.1770 - val_duration_loss: 1.2705\n",
      "Epoch 42/2000000\n",
      "22376/22376 [==============================] - 101s 5ms/step - loss: 0.5055 - pitch_loss: 0.4695 - duration_loss: 0.0360 - val_loss: 6.6874 - val_pitch_loss: 5.4064 - val_duration_loss: 1.2810\n",
      "Epoch 43/2000000\n",
      "22376/22376 [==============================] - 99s 4ms/step - loss: 0.4920 - pitch_loss: 0.4561 - duration_loss: 0.0358 - val_loss: 6.5841 - val_pitch_loss: 5.3238 - val_duration_loss: 1.2603\n",
      "Epoch 44/2000000\n",
      "22376/22376 [==============================] - 99s 4ms/step - loss: 0.4799 - pitch_loss: 0.4440 - duration_loss: 0.0359 - val_loss: 6.7462 - val_pitch_loss: 5.4418 - val_duration_loss: 1.3044\n",
      "Epoch 45/2000000\n",
      "22376/22376 [==============================] - 100s 4ms/step - loss: 0.4642 - pitch_loss: 0.4313 - duration_loss: 0.0329 - val_loss: 6.7804 - val_pitch_loss: 5.4563 - val_duration_loss: 1.3241\n",
      "Epoch 46/2000000\n",
      "22376/22376 [==============================] - 101s 5ms/step - loss: 0.4580 - pitch_loss: 0.4246 - duration_loss: 0.0334 - val_loss: 6.6945 - val_pitch_loss: 5.4460 - val_duration_loss: 1.2485\n",
      "Epoch 47/2000000\n",
      "22376/22376 [==============================] - 103s 5ms/step - loss: 0.4443 - pitch_loss: 0.4096 - duration_loss: 0.0347 - val_loss: 6.8306 - val_pitch_loss: 5.5016 - val_duration_loss: 1.3289\n",
      "Epoch 48/2000000\n",
      "22376/22376 [==============================] - 104s 5ms/step - loss: 0.4343 - pitch_loss: 0.4012 - duration_loss: 0.0331 - val_loss: 6.8631 - val_pitch_loss: 5.4902 - val_duration_loss: 1.3728\n",
      "Epoch 49/2000000\n",
      "22376/22376 [==============================] - 101s 5ms/step - loss: 0.4183 - pitch_loss: 0.3861 - duration_loss: 0.0322 - val_loss: 6.9033 - val_pitch_loss: 5.5763 - val_duration_loss: 1.3270\n",
      "Epoch 50/2000000\n",
      "22376/22376 [==============================] - 101s 5ms/step - loss: 0.4102 - pitch_loss: 0.3777 - duration_loss: 0.0325 - val_loss: 6.9685 - val_pitch_loss: 5.6347 - val_duration_loss: 1.3338\n",
      "Epoch 51/2000000\n",
      "22376/22376 [==============================] - 102s 5ms/step - loss: 0.4021 - pitch_loss: 0.3717 - duration_loss: 0.0304 - val_loss: 7.0304 - val_pitch_loss: 5.6788 - val_duration_loss: 1.3516\n",
      "Epoch 52/2000000\n",
      "22376/22376 [==============================] - 103s 5ms/step - loss: 0.3899 - pitch_loss: 0.3591 - duration_loss: 0.0308 - val_loss: 7.0565 - val_pitch_loss: 5.6773 - val_duration_loss: 1.3792\n",
      "Epoch 53/2000000\n",
      "22376/22376 [==============================] - 105s 5ms/step - loss: 0.3918 - pitch_loss: 0.3592 - duration_loss: 0.0326 - val_loss: 7.1002 - val_pitch_loss: 5.7021 - val_duration_loss: 1.3982\n",
      "Epoch 54/2000000\n",
      "22376/22376 [==============================] - 102s 5ms/step - loss: 0.3833 - pitch_loss: 0.3512 - duration_loss: 0.0320 - val_loss: 6.9196 - val_pitch_loss: 5.5622 - val_duration_loss: 1.3573\n",
      "Epoch 55/2000000\n",
      "22376/22376 [==============================] - 102s 5ms/step - loss: 0.3780 - pitch_loss: 0.3469 - duration_loss: 0.0311 - val_loss: 7.0688 - val_pitch_loss: 5.7256 - val_duration_loss: 1.3432\n",
      "Epoch 56/2000000\n",
      "22376/22376 [==============================] - 100s 4ms/step - loss: 0.3701 - pitch_loss: 0.3393 - duration_loss: 0.0308 - val_loss: 7.0650 - val_pitch_loss: 5.6855 - val_duration_loss: 1.3796\n",
      "Epoch 57/2000000\n",
      "22376/22376 [==============================] - 102s 5ms/step - loss: 0.3624 - pitch_loss: 0.3323 - duration_loss: 0.0302 - val_loss: 7.2489 - val_pitch_loss: 5.8612 - val_duration_loss: 1.3876\n",
      "Epoch 58/2000000\n",
      "22376/22376 [==============================] - 100s 4ms/step - loss: 0.3509 - pitch_loss: 0.3205 - duration_loss: 0.0304 - val_loss: 7.3740 - val_pitch_loss: 5.9587 - val_duration_loss: 1.4153\n",
      "Epoch 59/2000000\n",
      "22376/22376 [==============================] - 103s 5ms/step - loss: 0.3482 - pitch_loss: 0.3179 - duration_loss: 0.0303 - val_loss: 7.1120 - val_pitch_loss: 5.6889 - val_duration_loss: 1.4230\n",
      "Epoch 60/2000000\n",
      "22376/22376 [==============================] - 107s 5ms/step - loss: 0.3384 - pitch_loss: 0.3089 - duration_loss: 0.0295 - val_loss: 7.2612 - val_pitch_loss: 5.8386 - val_duration_loss: 1.4226\n",
      "Epoch 61/2000000\n",
      "22376/22376 [==============================] - 99s 4ms/step - loss: 0.3305 - pitch_loss: 0.3012 - duration_loss: 0.0293 - val_loss: 7.2158 - val_pitch_loss: 5.8056 - val_duration_loss: 1.4102\n",
      "Epoch 62/2000000\n",
      "22376/22376 [==============================] - 96s 4ms/step - loss: 0.3249 - pitch_loss: 0.2957 - duration_loss: 0.0292 - val_loss: 7.2023 - val_pitch_loss: 5.7695 - val_duration_loss: 1.4328\n",
      "Epoch 63/2000000\n",
      "22376/22376 [==============================] - 96s 4ms/step - loss: 0.3217 - pitch_loss: 0.2897 - duration_loss: 0.0320 - val_loss: 7.4269 - val_pitch_loss: 5.9262 - val_duration_loss: 1.5007\n",
      "Epoch 64/2000000\n",
      "22376/22376 [==============================] - 96s 4ms/step - loss: 0.3247 - pitch_loss: 0.2938 - duration_loss: 0.0309 - val_loss: 7.1732 - val_pitch_loss: 5.7189 - val_duration_loss: 1.4543\n",
      "Epoch 65/2000000\n",
      "22368/22376 [============================>.] - ETA: 0s - loss: 0.3172 - pitch_loss: 0.2885 - duration_loss: 0.0286"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-48-832a0ff938cf>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     35\u001b[0m           \u001b[0;34m,\u001b[0m \u001b[0mvalidation_split\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m           \u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks_list\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 37\u001b[0;31m           \u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     38\u001b[0m          )\n\u001b[1;32m     39\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.virtualenvs/gdl/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1037\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1038\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1039\u001b[0;31m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1040\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1041\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m~/.virtualenvs/gdl/lib/python3.6/site-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m    210\u001b[0m                         val_outs = test_loop(model, val_f, val_ins,\n\u001b[1;32m    211\u001b[0m                                              \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 212\u001b[0;31m                                              verbose=0)\n\u001b[0m\u001b[1;32m    213\u001b[0m                         \u001b[0mval_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_outs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    214\u001b[0m                         \u001b[0;31m# Same labels assumed.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.virtualenvs/gdl/lib/python3.6/site-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mtest_loop\u001b[0;34m(model, f, ins, batch_size, verbose, steps)\u001b[0m\n\u001b[1;32m    390\u001b[0m                 \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    391\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 392\u001b[0;31m             \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    393\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    394\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mbatch_index\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.virtualenvs/gdl/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2713\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2714\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2715\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2716\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2717\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.virtualenvs/gdl/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2673\u001b[0m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_metadata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2674\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2675\u001b[0;31m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2676\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.virtualenvs/gdl/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1437\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[1;32m   1438\u001b[0m               \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1439\u001b[0;31m               run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1440\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1441\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "weights_folder = os.path.join(run_folder, 'weights')\n",
    "\n",
    "checkpoint1 = ModelCheckpoint(\n",
    "    os.path.join(weights_folder, \"weights-improvement-{epoch:02d}-{loss:.4f}-bigger.h5\"),\n",
    "    monitor='loss',\n",
    "    verbose=0,\n",
    "    save_best_only=True,\n",
    "    mode='min'\n",
    ")\n",
    "\n",
    "checkpoint2 = ModelCheckpoint(\n",
    "    os.path.join(weights_folder, \"weights.h5\"),\n",
    "    monitor='loss',\n",
    "    verbose=0,\n",
    "    save_best_only=True,\n",
    "    mode='min'\n",
    ")\n",
    "\n",
    "early_stopping = EarlyStopping(\n",
    "    monitor='loss'\n",
    "    , restore_best_weights=True\n",
    "    , patience = 10\n",
    ")\n",
    "\n",
    "\n",
    "callbacks_list = [\n",
    "    checkpoint1\n",
    "    , checkpoint2\n",
    "    , early_stopping\n",
    " ]\n",
    "\n",
    "model.save_weights(os.path.join(weights_folder, \"weights.h5\"))\n",
    "model.fit(network_input, network_output\n",
    "          , epochs=2000000, batch_size=32\n",
    "          , validation_split = 0.2\n",
    "          , callbacks=callbacks_list\n",
    "          , shuffle=True\n",
    "         )\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "gdl",
   "language": "python",
   "name": "gdl"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}