NLP_PS4_HyperParameterTuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0512 22:52:30.576318 13204 __init__.py:56] Some hub symbols are not available because TensorFlow version is less than 1.14\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import operator\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import csv\n",
    "import re\n",
    "\n",
    "from itertools import chain, islice, product\n",
    "from functools import partial\n",
    "\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV, ShuffleSplit\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "import keras\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Layer, Lambda, Input, InputLayer, Embedding, Dense, TimeDistributed, Flatten, Reshape, Activation, Dropout, BatchNormalization, CuDNNLSTM, LSTM, Bidirectional, Multiply\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras.models import Model\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras import backend as K\n",
    "\n",
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from pprint import pprint\n",
    "\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_seed(seed):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    tf.set_random_seed(seed)\n",
    "    \n",
    "set_seed(1992)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Preprocessing</h2>\n",
    "\n",
    "We operate on a subset of two datasets:\n",
    "<ol>\n",
    "    <li>\n",
    "        <h4>LingSpam Dataset (Bare)</h4>\n",
    "        <p>\n",
    "            This is the dataset that we used in the 2nd assignment.\n",
    "            It consists of a few large emails, appriximately 1000 messages with an average of 3500 words.\n",
    "    </li>\n",
    "    <li>\n",
    "        <h4>Kaggles Spam/Ham Dataset</h4>\n",
    "        <p>\n",
    "            This dataset is bigger, consisting of approximately 5500 messages.\n",
    "            Each message however is quiet short and a lot of jargon is used in them.\n",
    "        </p>\n",
    "    </li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "lingspam_data_path = \"lingspam_public/bare\"\n",
    "kaggle_data_path   = \"Kaggle/kaggle_spam_data.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_lingspam_data(path):\n",
    "\n",
    "    ham, spam = [], []\n",
    "\n",
    "    for dirpath, _, filenames in os.walk(path):\n",
    "        for filename in filenames:\n",
    "            with open(os.path.join(dirpath, filename), encoding=\"utf8\") as file:\n",
    "                message = file.read()\n",
    "            if filename.startswith(\"spm\"):\n",
    "                spam.append(message)\n",
    "            else:\n",
    "                ham.append(message)\n",
    "\n",
    "    return ham, spam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_kaggle_data(path):\n",
    "    \n",
    "    ham, spam = [], []\n",
    "    \n",
    "    pattern = re.compile(\"\\\"?(?P<Category>spam|ham),(?:\\\"\\\")?(?P<Text>.*?)(?:\\\"\\\"\\\")?;+\")\n",
    "    \n",
    "    with open(path, 'r', encoding=\"utf8\") as file:\n",
    "        for line in file.readlines():\n",
    "            match = pattern.match(line)\n",
    "            if match:\n",
    "                text = \" \".join(word_tokenize(match[\"Text\"]))\n",
    "                if match[\"Category\"].startswith(\"s\"):\n",
    "                    spam.append(text)\n",
    "                else:\n",
    "                    ham.append(text)\n",
    "            \n",
    "    return ham, spam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ham: 2412, Spam: 481\n"
     ]
    }
   ],
   "source": [
    "dataset = \"l\" # Specify the dataset to be loaded: kaggle, lingspam, or both\n",
    "\n",
    "ham, spam = \\\n",
    "    load_kaggle_data(kaggle_data_path) if dataset.startswith(\"k\") else (\n",
    "        load_lingspam_data(lingspam_data_path) if dataset.startswith(\"l\") else\n",
    "        tuple(map(operator.concat, load_kaggle_data(kaggle_data_path), load_lingspam_data(lingspam_data_path)))\n",
    "    )\n",
    " \n",
    "print(\"Ham: {}, Spam: {}\".format(len(ham), len(spam)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_split(x, y, sizes=(0.1, 0.1)):\n",
    "    \n",
    "    data_size = len(x)\n",
    "    splits = []\n",
    "    \n",
    "    for size in sizes:\n",
    "        x, test_x, y, test_y = train_test_split(x, y, test_size=int(data_size*size), shuffle=True)\n",
    "        splits.append((test_x, test_y))\n",
    "    splits.append((x, y))\n",
    "    \n",
    "    return splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split Sizes: [2084, 231, 578]\n"
     ]
    }
   ],
   "source": [
    "x = ham + spam\n",
    "y = [0] * len(ham) + [1] * len(spam)\n",
    "\n",
    "(test_x, test_y), (valid_x, valid_y), (train_x, train_y) = data_split(x, y, sizes=(0.2, 0.08))\n",
    "\n",
    "print(\"Split Sizes: {}\".format(list(map(len, (train_x, valid_x, test_x)))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Tokenizer Initialization</h3>\n",
    "<p>We consider two tokenizers. <i>Tokenizer_TRAIN_GLB</i> is fitted in all tokens onservable in the training set. <i>Tokenizer_UNK</i> is fitted in a subset of the training sequences in order to simulate the unobservability of a number of tokens in the validation data.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokens discovered by Tokenizer_GLB: 50155\n"
     ]
    }
   ],
   "source": [
    "tokenizer_GLB = Tokenizer(num_words=None, lower=True, oov_token=\"*UNK*\")\n",
    "tokenizer_GLB.fit_on_texts(train_x)\n",
    "\n",
    "print(\"Tokens discovered by Tokenizer_GLB: {}\".format(len(tokenizer_GLB.word_index)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokens discovered by Tokenizer_UNK: 47449\n"
     ]
    }
   ],
   "source": [
    "tokenizer_UNK = Tokenizer(num_words=None, lower=True, oov_token=\"*UNK*\")\n",
    "tokenizer_UNK.fit_on_texts(random.sample(train_x, len(train_x) - len(valid_x))) # We want the number of unknown words in the train set to be approximately the same as in the validation set.\n",
    "\n",
    "print(\"Tokens discovered by Tokenizer_UNK: {}\".format(len(tokenizer_UNK.word_index)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Tokenization</h3>\n",
    "<p>We transform each set in sequences of tokens according the chosen tokenizer.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = tokenizer_UNK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sequences = pad_sequences(tokenizer.texts_to_sequences(train_x), maxlen=150, padding=\"post\", truncating=\"post\") # We are constrained by the hardware limitations.\n",
    "input_length = len(train_sequences[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_sequences = pad_sequences(tokenizer.texts_to_sequences(valid_x), maxlen=input_length, padding=\"post\", truncating=\"post\")\n",
    "test_sequences  = pad_sequences(tokenizer.texts_to_sequences(test_x), maxlen=input_length, padding=\"post\", truncating=\"post\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Utilities</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_rnn_model(rnn_stack_size=1, use_atention=False, mlp_model=None):\n",
    "    assert rnn_stack_size > 0\n",
    "    \n",
    "    l_Input = Input((input_length,), name=\"In\")\n",
    "\n",
    "    l_Embeddings = Embedding(\n",
    "        name=\"Embeddings\",\n",
    "        input_dim=embedding_weights.shape[0],\n",
    "        output_dim=embedding_weights.shape[1],\n",
    "        input_length=input_length,\n",
    "        embeddings_initializer=\"zeros\",\n",
    "        weights=[embedding_weights],\n",
    "        trainable=False\n",
    "    )(l_Input)\n",
    "\n",
    "    m_BiLSTM = Sequential(name=\"BiLSTM_Stack\")\n",
    "    for _ in range(rnn_stack_size - 1):\n",
    "        m_BiLSTM.add(Bidirectional(CuDNNLSTM(100, return_sequences=True)))\n",
    "        \n",
    "    if use_atention:\n",
    "        m_BiLSTM.add(Bidirectional(CuDNNLSTM(100, return_sequences=True)))\n",
    "    else:\n",
    "        m_BiLSTM.add(Bidirectional(CuDNNLSTM(100)))\n",
    "        \n",
    "        \n",
    "    l_BiLSTM = m_BiLSTM(l_Embeddings)\n",
    "\n",
    "    m_Out = Sequential(name=\"Out\")\n",
    "    \n",
    "    if use_atention:\n",
    "\n",
    "        m_Attention = Sequential(name=\"Attention\")\n",
    "        m_Attention.add(mlp_model)\n",
    "        m_Attention.add(Flatten())\n",
    "        m_Attention.add(Activation(\"softmax\"))\n",
    "        m_Attention.add(Reshape((input_length, 1)))\n",
    "        l_Attention = m_Attention(l_BiLSTM)\n",
    "\n",
    "        l_Multiply = Multiply(name=\"Multiply\")([l_BiLSTM, l_Attention])\n",
    "\n",
    "        m_Out.add(Lambda(lambda x: K.sum(x, axis=1)))\n",
    "        m_Out.add(Dense(1, activation=\"sigmoid\"))\n",
    "        l_Out = m_Out(l_Multiply)\n",
    "    \n",
    "    else:\n",
    "        \n",
    "        m_Out.add(Dense(1, activation=\"sigmoid\"))\n",
    "        l_Out = m_Out(l_BiLSTM)\n",
    "\n",
    "    model = Model(inputs=l_Input, outputs=l_Out)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_model(model):\n",
    "    \n",
    "    K.clear_session()\n",
    "    del model\n",
    "    gc.collect() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tune_model(build_fn, param_grid={}, epochs=100, patience=6, verbose=False):\n",
    "\n",
    "    tune_history = []\n",
    "    best_score = np.inf\n",
    "    best_index = None\n",
    "    \n",
    "    param_iter = product(*param_grid.values())\n",
    "    for params in param_iter:\n",
    "\n",
    "        model = build_fn(**dict(zip(param_grid.keys(), params)))\n",
    "        model.summary()\n",
    "\n",
    "        model.compile(optimizer=\"adam\", loss=\"binary_crossentropy\", metrics=[\"accuracy\"])\n",
    "\n",
    "        history = model.fit(\n",
    "            train_sequences, train_y,\n",
    "            validation_data=(valid_sequences, valid_y),\n",
    "            batch_size=16,\n",
    "            epochs=epochs,\n",
    "            callbacks=[EarlyStopping(patience=patience, restore_best_weights=True, verbose=verbose)],\n",
    "            verbose=verbose\n",
    "        )\n",
    "        \n",
    "        val_loss = np.min(history.history[\"val_loss\"])\n",
    "        if val_loss < best_score:\n",
    "            best_score = val_loss\n",
    "            best_index = len(tune_history)\n",
    "            \n",
    "        tune_history.append({\n",
    "            \"params\": params,\n",
    "            \"model\": model,\n",
    "            \"fit_history\": history\n",
    "        })\n",
    "        \n",
    "    return tune_history, best_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_model(model, sequences, y):\n",
    "    \n",
    "    print(classification_report(y, (model.predict(sequences).flatten() >= 0.5).astype(float)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_history(history):\n",
    "    \n",
    "    plt.plot(\n",
    "        history.history[\"acc\"],\n",
    "        label=\"Train\"\n",
    "    )\n",
    "    plt.plot(\n",
    "        history.history[\"val_acc\"],\n",
    "        label=\"Validation\"\n",
    "    )\n",
    "    \n",
    "    plt.title(\"Model Accuracy\")\n",
    "    plt.ylabel(\"Accuray\")\n",
    "    plt.xlabel(\"Epoch\")\n",
    "    plt.legend([\"Train\", \"Validation\"], loc=\"best\")\n",
    "    \n",
    "    plt.show() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>BiLSTM on Pretrained (Glove) Embeddings</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_embeddings_path = \"glove.6B/glove.6B.100d.txt\"\n",
    "embeddings_df = pd.read_csv(glove_embeddings_path, sep=\" \", index_col=0, header=None, quoting=csv.QUOTE_NONE, na_values=None, keep_default_na=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Non-Zero Embeddings: 32081\n"
     ]
    }
   ],
   "source": [
    "token_iter = map(\n",
    "    operator.itemgetter(0),\n",
    "    sorted(\n",
    "        filter(lambda item: item[1] <= tokenizer.num_words, tokenizer.word_index.items()) if tokenizer.num_words\n",
    "            else tokenizer.word_index.items(),\n",
    "        key=operator.itemgetter(1)\n",
    "    )\n",
    ")\n",
    "\n",
    "embedding_weights = embeddings_df.reindex(chain((None,), token_iter), fill_value=0.0).values # The zero-indexed vector is used only for masking by the layer.\n",
    "\n",
    "print(\"Non-Zero Embeddings: {}\".format(np.count_nonzero(embedding_weights, axis=0)[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\josep\\Anaconda3\\envs\\NLP\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0512 22:53:32.936457 13204 deprecation.py:323] From C:\\Users\\josep\\Anaconda3\\envs\\NLP\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "In (InputLayer)              (None, 150)               0         \n",
      "_________________________________________________________________\n",
      "Embeddings (Embedding)       (None, 150, 100)          4745000   \n",
      "_________________________________________________________________\n",
      "BiLSTM_Stack (Sequential)    (None, 200)               403200    \n",
      "_________________________________________________________________\n",
      "Out (Sequential)             (None, 1)                 201       \n",
      "=================================================================\n",
      "Total params: 5,148,401\n",
      "Trainable params: 403,401\n",
      "Non-trainable params: 4,745,000\n",
      "_________________________________________________________________\n",
      "WARNING:tensorflow:From C:\\Users\\josep\\Anaconda3\\envs\\NLP\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0512 22:53:36.480271 13204 deprecation.py:323] From C:\\Users\\josep\\Anaconda3\\envs\\NLP\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 2084 samples, validate on 231 samples\n",
      "Epoch 1/30\n",
      "2084/2084 [==============================] - 7s 3ms/step - loss: 0.1622 - acc: 0.9395 - val_loss: 0.0367 - val_acc: 0.9957\n",
      "Epoch 2/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0992 - acc: 0.9645 - val_loss: 0.0343 - val_acc: 0.9913\n",
      "Epoch 3/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0510 - acc: 0.9856 - val_loss: 0.0341 - val_acc: 0.9827\n",
      "Epoch 4/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0625 - acc: 0.9808 - val_loss: 0.1920 - val_acc: 0.9307\n",
      "Epoch 5/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0337 - acc: 0.9894 - val_loss: 0.0075 - val_acc: 1.0000\n",
      "Epoch 6/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0257 - acc: 0.9904 - val_loss: 0.0096 - val_acc: 1.0000\n",
      "Epoch 7/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.1277 - acc: 0.9592 - val_loss: 0.0160 - val_acc: 0.9957\n",
      "Epoch 8/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0332 - acc: 0.9885 - val_loss: 0.0096 - val_acc: 1.0000\n",
      "Epoch 9/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0280 - acc: 0.9885 - val_loss: 0.0061 - val_acc: 1.0000\n",
      "Epoch 10/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0149 - acc: 0.9966 - val_loss: 0.0058 - val_acc: 1.0000\n",
      "Epoch 11/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0133 - acc: 0.9962 - val_loss: 0.0062 - val_acc: 0.9957\n",
      "Epoch 12/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0026 - acc: 0.9990 - val_loss: 0.0417 - val_acc: 0.9827\n",
      "Epoch 13/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0034 - acc: 0.9990 - val_loss: 0.0217 - val_acc: 0.9913\n",
      "Epoch 14/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0085 - acc: 0.9957 - val_loss: 0.0021 - val_acc: 1.0000\n",
      "Epoch 15/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0033 - acc: 0.9976 - val_loss: 0.0016 - val_acc: 1.0000\n",
      "Epoch 16/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0015 - acc: 0.9995 - val_loss: 0.0010 - val_acc: 1.0000\n",
      "Epoch 17/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 3.9199e-04 - acc: 1.0000 - val_loss: 7.7545e-04 - val_acc: 1.0000\n",
      "Epoch 18/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 2.4431e-04 - acc: 1.0000 - val_loss: 5.9623e-04 - val_acc: 1.0000\n",
      "Epoch 19/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 1.8384e-04 - acc: 1.0000 - val_loss: 4.9298e-04 - val_acc: 1.0000\n",
      "Epoch 20/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 1.4481e-04 - acc: 1.0000 - val_loss: 4.1804e-04 - val_acc: 1.0000\n",
      "Epoch 21/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 1.1786e-04 - acc: 1.0000 - val_loss: 3.6320e-04 - val_acc: 1.0000\n",
      "Epoch 22/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 9.7812e-05 - acc: 1.0000 - val_loss: 3.1852e-04 - val_acc: 1.0000\n",
      "Epoch 23/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 8.3168e-05 - acc: 1.0000 - val_loss: 2.9050e-04 - val_acc: 1.0000\n",
      "Epoch 24/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 6.9946e-05 - acc: 1.0000 - val_loss: 2.5342e-04 - val_acc: 1.0000\n",
      "Epoch 25/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 6.1002e-05 - acc: 1.0000 - val_loss: 2.2773e-04 - val_acc: 1.0000\n",
      "Epoch 26/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 5.3209e-05 - acc: 1.0000 - val_loss: 2.1794e-04 - val_acc: 1.0000\n",
      "Epoch 27/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 4.7006e-05 - acc: 1.0000 - val_loss: 2.0037e-04 - val_acc: 1.0000\n",
      "Epoch 28/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 4.1965e-05 - acc: 1.0000 - val_loss: 1.9109e-04 - val_acc: 1.0000\n",
      "Epoch 29/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 3.7384e-05 - acc: 1.0000 - val_loss: 1.8070e-04 - val_acc: 1.0000\n",
      "Epoch 30/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 3.3556e-05 - acc: 1.0000 - val_loss: 1.6877e-04 - val_acc: 1.0000\n"
     ]
    }
   ],
   "source": [
    "param_grid = {\n",
    "    \"use_atention\": [False],\n",
    "    \"rnn_stack_size\": [2],\n",
    "    \"mlp_model\": [None]\n",
    "}\n",
    "\n",
    "history, best_index = tune_model(create_rnn_model, param_grid=param_grid, epochs=30, patience=6, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tune Results: #1, {'use_atention': False, 'rnn_stack_size': 2, 'mlp_model': None}\n"
     ]
    }
   ],
   "source": [
    "print(\"Tune Results: #{}, {}\".format(best_index + 1, dict(zip(param_grid.keys(), history[best_index][\"params\"]))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8lPW1+PHPyb4QyMaasIMVtAiK4A6uV7QVt6pY6y6t2mvbe72tXX622nrb29rWtnp7ixUVi1JLq7WtaylupYoBAQWMIAQIawiQfc/5/fE8EybDJDOTmSczCef9euXFzLN+Hwbm5LseUVWMMcaY7iTFuwDGGGMSnwULY4wxIVmwMMYYE5IFC2OMMSFZsDDGGBOSBQtjjDEhWbAwRzURGSMiKiIpYRx7o4i83RvlMibRWLAwfYaIlIlIs4gUBmxf437hj4lPyTqVJVtEakXkxXiXxZhYsmBh+pqtwDzfGxH5NJAZv+Ic4UqgCbhARIb35o3DqR0Z01MWLExf8xRwvd/7G4BF/geIyCARWSQiFSKyTUS+IyJJ7r5kEXlQRPaLyBbg4iDnPiYiu0Vkp4j8QESSIyjfDcD/AeuAzwdce6SI/MktV6WIPOy37zYR2SgiNSKyQUROdLeriEzwO+4JEfmB+3q2iJSLyDdEZA/wuIjkichf3XscdF8X+52fLyKPi8gud//z7vYPReSzfselun9HUyN4dtOPWbAwfc07wEARmeR+iV8N/C7gmF8Bg4BxwCyc4HKTu+824DPANGA6Tk3A35NAKzDBPeYC4NZwCiYio4DZwGL353q/fcnAX4FtwBigCFji7vsc8D33+IHAJUBlOPcEhgH5wGhgPs7/6cfd96OABuBhv+OfArKA44AhwM/d7YuA6/yOuwjYraprwiyH6e9U1X7sp0/8AGXAecB3gB8CFwKvASmA4nwJJ+M0A032O++LwOvu638AX/Lbd4F7bgow1D0302//PGC5+/pG4O1uyvcdYI37egTQBkxz358KVAApQc57BfhKF9dUYILf+yeAH7ivZwPNQEY3ZZoKHHRfDwfagbwgx40AaoCB7vulwNfj/ZnbT+L8WBun6YueAt4ExhLQBAUUAmk4v8H7bMP5TR6cL8UdAft8RgOpwG4R8W1LCji+O9cDjwKo6i4ReQOnWep9YCSwTVVbg5w3EvgkzHsEqlDVRt8bEcnCqS1cCOS5m3Pcms1I4ICqHgy8iFvefwJXiMhzwBzgKz0sk+mHrBnK9Dmqug2no/si4E8Bu/cDLThf/D6jgJ3u6904X5r++3x24NQsClU11/0ZqKrHhSqTiJwGTAS+KSJ73D6EmcA8t+N5BzCqi07oHcD4Li5dj9Ns5DMsYH/gstH/CXwKmKmqA4GzfEV075MvIrld3OtJnKaozwH/UtWdXRxnjkIWLExfdQtwjqrW+W9U1TbgWeABEckRkdHAf3C4X+NZ4C4RKRaRPOAev3N3A68CPxWRgSKSJCLjRWRWGOW5AadJbDJO089U4HicL/o5wEqcQPUjd3hthoic7p77W+BuETlJHBPccgOsAa51O+YvxOmD6U4OTj/FIRHJB74b8HwvAf/rdoSnishZfuc+D5yIU6MIrLGZo5wFC9MnqeonqlrSxe5/B+qALcDbwNPAQnffozh9BGuB1RxZM7kepxlrA3AQp+2+2yGwIpIBXAX8SlX3+P1sxWkyu8ENYp/F6TjfDpTjdM6jqn8AHnDLWYPzpZ3vXv4r7nmHcEZXPd9dWYCHcIYS78cZDPBywP4v4NS8PgL2AV/17VDVBuCPOM17gX8v5ignqpb8yBjjEJF7gWNU9bqQB5ujinVwG2MAZw4GTvPeF+JdFpN4rBnKGIOI3IbTAf6Sqr4Z7/KYxGPNUMYYY0KymoUxxpiQ+k2fRWFhoY4ZMybexTDGmD5l1apV+1V1cKjj+k2wGDNmDCUlXY2kNMYYE4yIbAt9lDVDGWOMCYMFC2OMMSFZsDDGGBOSBQtjjDEhWbAwxhgTkmfBQkQWisg+Efmwi/0iIr8Ukc0iss6XRtLdd4OIbHJ/bvCqjMYYY8LjZc3iCZwELF2Zg7P+/0ScdJC/ho71ab6LkwtgBvBddylpY4wxceLZPAtVfVNExnRzyFxgkTrrjbwjIrkiMhwnVeRrqnoAQERewwk6z3hV1j5BFUoeg5q94R0//ASY9JnY3X/vetjwZ6ccsTRiGhx7UWyvGWu71sBHf+u126kqtc2tVDW0Ut3QQk1jK7Ysj+mODCpi5uf+09N7xHNSXhGd01WWu9u62n4EEZmPUyth1KhRwQ7pP6p3wd98/xik20M7kqdd/luY8rno771/Mzx+ETQeCuPekXDLeeXjcPzlMbxuDKnC83fAvvXE8tkDv/o14E02zs+ImN3R9Geb9nwKJ0mid+IZLIL9z9Nuth+5UXUBsABg+vTp/ftXr6Zq58/PPQHHXdb9sa3N8NRl8Oc7IHcUjJrZ8/vWH4Cnr4KkZLhrDeSP7fm1ArU0wqK58PztTjmLp8fu2rGybYUTKC75FZx4fdSXa2lr59mSHfxq2Wb2VHekziY3K5VR+VmMzMtiZH4WI/MzO96PyM0kLcXGopiufaoX7hHPYFFO51zIxcAud/vsgO2v91qpElVTjfNnek7oY1PS4Oqn4LfnwZJr4da/9+xLvrUZfv8FqNoBN/wltoECIDUDrlkMvz0XnrkGbvuHEzR6qL1d+c2bW1i2MbymurSUJP79nImcOr6g64NW/gYy8+DT0dXQ2tqVF9bu5OevbWL7gXpOGp3Hty+exLjB2YzMz2JgRmpU1zfGa/H8deUF4Hp3VNQpQJWbI/gV4AI3R3AecIG77ejmq1mkhREsALLy4dpnob0Vnr4aGg5Fdj9V+OvXYNvbMPd/YdQpkZ0fruxCp5ytzU45G6t7dJmq+hZuefI9/uflj2hpayc9NSnkz7bKem5YuJKXP9zdxUXLYeNfnRpFamaPyqWqvPzhHub84k2+9vu1DEhPYeGN01n6pVP57AkjOG7EIAsUpk/wrGYhIs/g1BAKRaQcZ4RTKoCq/h/wInARsBmoB25y9x0Qke8D77mXut/X2X1Ua6p1/gynZuFTOAGu/p3TJPWHG+Hzf4DkML+Y3v45rPkdzLonNv0e3Rn8KbjqSfjdFbD0Jpj3e0gO/5/m+l1V3P671eyuauD+ucfxhVNGIxK6f+FQfTM3P/EedyxezQ8u/TTXzgyo1ZQsBBSm3xLhAzlB4q1N+/npq6WsLa9i3OBsHr52GhcdP5ykpFj2+xjTO7wcDTUvxH4F7uxi30JgoRfl6rMiaYbyN/ZM+OxD8Oc74cX/gs/8HEJ9ka5/HpbdB8dfCbPv6Vl5IzX+bPjMz+AvX4FXvgkX/SSs0/60upxv/ukDcrNSWTL/VE4aHf4o69ysNH5360zuWLyabz33AZW1TXz5nAlOoGlphFVPwDFzIG90RI9SUnaAn7xSyrtbD1CUm8mPr5zC5dOKSEm2fgfTd/WbJcr7vWZfzWJA5OdOuw4qNzu1hcKJcGrQGO3YuQqe+yIUz4C5j4QOLLF00o2wfxP862EomAAzv9jloc2t7fzgbxtY9K9tzBybz8PXnsjgnPSIb5mVlsKj10/n60vX8dPXPqayrpl7PzOZpPXPQX0lzJwf9rU+3FnFT18tZXlpBYUD0rnvkuO4ZsZI0lOSIy6XMYnGgkVf4atZhNtnEeice6HyE3jl25A3NvjchkM74Jl5MGAIXPO00wHd286/Hw5shZfvccp5zAVHHLKnqpE7Fq9i9fZD3HbmWL5x4bFR/daempzETz93AgXZafz27a0cqG3ioZrfkFT4KRg7K+T5m/fV8vPXPuZvH+xmUGYq37jwWG44bTRZafbfy/Qf9q+5r2iqgZTMiNryO0lKgst+43Ta/vFWuPklZ+Ke//WfuQZaGuD6F2BAyMRZ3khKhisehYUXOv0XN78Cw47v2P3Olkq+/PRq6pvbeOTaE7l4yvDY3DZJ+PbFkyjMSeeVl/9CUvr7NF3wY9K7qVmVH6znF3/fxB9Xl5ORmsxd50zgljPHMSjTOqxN/2PBoq9oqom8vyJQWhbMWwKPngNPXwO3LYOBI6C9DZbeAvs2wnVLYcixsSlzCL9+/RN+9842ivMyGZmf5cwryM9kZF4WYz67iIIlc5Cnr4bb/oEOGMJjb2/lhy99xOiCLJ657RQmDo3y7yOAiPClWeOZU7qSmp2Z3Lp6PL8+oZn87LROx+2raeSRf2zm6ZXbERFuOn0st88eT+GAyJvBjOkrLFj0FbEIFgA5Q+Ha38PCf3NqEje9BMu+D5tecTq/x58T/T3CUN3YwiPLNzNsUAbtqry1qYK91U2djpmWchdPp3yPXb+6hO8P/jGvf1LLhccN4yefm0KOV8NNa/Yyes+rbDtmHms2tnDl/63gqVtmUpSbyaH6Zv7vjS08sWIrLW3KVdNHcte5Exg+qGfDao3pSyxYAOxcDcOm9LyJpzc01/asczuYYcc7S2w8czUsOBv2l8Ipd8L0m2Nz/TAsWbmd2qZWHrp6KscXDQKgsaWN8oMN7DhYT/mBerYfGMsTO77DF/fcy3W7f8gpF/6KL86aENaw2B5b9QS0tzD6wq+y6NRcbl1UwpW/XsFl04p46l/bqG1uZe4JI/jqeccwpjDbu3IYk2AS+Nuxl1R87Mx0PvlWuOjH8S5N15pqIH1g7K53zAVw4Y/gpa87w0Mv+H7srh1CS1s7j/+zjFPG5XcECoCM1GQmDBnAhCH+QXEyrBDOe/U7nJe/BmSidwVrbXbmVkw4DwrGM7MAnv3iqVy/cCX/+/onXDB5KP9xwTEcOyyGn4MxfYQFi8HHwMwvwTuPuMM1wx8q2auaamBg0PUUe27mF51VX4dNcTqWe8nf1u1md1UjD1x2fOiDAU79Mqz4FZS+6O0EwY/+ArV7YMavOjZNGj6Qv911BpW1zUwabkHCHL0sWIDzW/WBLfDyN5z1jyaeH+8SHSlWfRaBRs6I/TW7oaoseHML4wdnM/uYIeGdJOL8tv/R35zOeK8C27sLIH+ccy8/Q3IyGJITh2HExiQQm1IK7nDN38LQ4+APN8HeDfEu0ZGaamLXZxFH//qkkg27q7ntzHGRLXsx4VxnifSdq7wp2O61sOMdOPk2Z5ixMaYT+1/hkz7AWZMoLdtZkjvcJEO9pbnWm5pFL3v0rS0UDkjj0mkRNqmNOxskCTb/3ZuCvbsAUrNg6rXeXN+YPs6Chb9BRXDtEmeZhyXXOhPUEkFbC7Q29vlgsWlvDctLK7j+1DFkpEbYlJSVD0XTYdNrsS9Y/QH44A9wwjWQmRv76xvTD1iwCDRiGlz+qLtG0pegvT3eJYp+qY8E8du3tpKRmsR1p0S2MF+HiefDrvehbn9sC7b6SWhrghkJOrjBmARgwSKYSZ+B8++DDc/D8gfiXZqerzibQPbVNPLc+zu58qTiI2ZEh23CuYDCJ/+IXcHaWuG9x2DMmTBkUuyua0w/Y8GiK6fd5SS9eetBWPNMfMvSESz6bgf3U//aRkt7O7ecMa7nFxk+DbIKYttv8fFLTibAbla4NcZYsOiaCFz8Mxh7Frzw71D2z/iVpbkHiY8SSENzG0+9s43zJg1lbDSznpOSYPy5sHlZ7JoHVy6AQSOdiYnGmC5ZsOhOcipctQjyxsDvP+8s8R0PHTWLvjkpbOmqHRyqb2H+WVHUKnwmng/1+2H3muivtW8jbH0TTr4lsZd6MSYBWLAIJTMPPv8sIM6Q2oaDvV+Gjg7uvtcM1dauPPb2Vk4Ymcv0CLLYdWn8OYDEpilq5QJITodp10d/LWP6OQsW4cgf5yQDOrTdWcq7t/XhDu7XNuylrLKe+WeOi80CgNmFzoi1aINFwyFY+3v49OcguyD6chnTz1mwCNfoU+Hc78Iny6Dco1nEXenDHdy/fWsLxXmZ/NtxQ2N30QnnQfl7zvyInlrzNLTUJe5aYMYkGE+DhYhcKCKlIrJZRO4Jsn+0iCwTkXUi8rqIFPvt+x8R+dD9udrLcobtpBucuQ4rf9O79/V1cIfZDNXc2k5za/znh6zefpCSbQe55YyxUaU9PcLE80HbYcvrPTu/vR3eexRGzuycLdAY0yXPgoWIJAOPAHOAycA8EZkccNiDwCJVnQLcD/zQPfdi4ERgKjAT+C8RiX/vbnqOsxzEh3+C2n29d9+mGidQdLGAXnu78uHOKn7zxidcv3AlJ9z3KnMfiePoLddv39rCwIwUrpo+MrYXLjoJMnJ73hS1+e/OwpE2Cc+YsHk5BGQGsFlVtwCIyBJgLuC/St9k4Gvu6+XA837b31DVVqBVRNYCFwLPelje8My4zalZrHoSZv1X79zTFyxcqsqW/XWs2Lyff26u5J2tlRyqbwFgwpABjCnMpnRPNa1t7bH9jT4C2yvrefnDPcw/azzZ6TH+Z5aU7HR0b/47qDrDnCOxcgEMGAaT58a2XMb0Y14GiyJgh9/7cpxagr+1wBXAL4DLgBwRKXC3f1dEfgZkAWfTOcgAICLzgfkAo0aNinX5gyuc6HxRlTwGZ3zVGV7rNXd58tc27OWlD3ezYnMle6obASjKzeT8SUM5fUIhp40vYMjADJ5ZuZ1v/ukD9lQ3UpyX5X35glj4z60kJwk3njbGmxtMOA/W/wn2fADDp4R/XuUnsPk1mP2t3vnsjOknvAwWwX7d04D3dwMPi8iNwJvATqBVVV8VkZOBFUAF8C+g9YiLqS4AFgBMnz498NremfFFJyXpxr/A8Zd7f7+mGuokk9sWlZCfncap4wo4bUIBp48vZHRB1hGjjIpynZzQuw7FJ1gcqm/m2ZIdfPaEEQwb5FEeiAnnOn9u/ntkwWLlo5CUCifd6EmxjOmvvAwW5YB/Y3UxsMv/AFXdBVwOICIDgCtUtcrd9wDwgLvvaWCTh2WNzMTznYl6Kx/tnWDRXEt5fQo5GSm89fWzQzbrjOgIFvFZNXfxu9upb27jtjNjMAmvKznDYNinnWBx5n+Ed05TLaxZDMddCjkxHJ1lzFHAy2DxHjBRRMbi1BiuATolCxCRQuCAqrYD3wQWutuTgVxVrRSRKcAU4FUPyxqZpGQnZ/er33GaQYZ92tPbtdRXsa02i8+dPDKs9v8Ruc5v8ztjGCze2VLJkpXbwzr2jY8rOHNiofdpSCec56RbbayCjEGhj1+3BJqqnZqhMSYingULVW0VkS8DrwDJwEJVXS8i9wMlqvoCMBv4oYgoTjPUne7pqcBbbvNKNXCd29mdOKZdB8v/G979Dcx92NNb1dccolbzuf7U8Jb2zkpLIS8rNaY1iydXlLFs4z6G54ZuVsrPTuOr502M2b27NOF8ePvnsOUNmHxJ98eqOjXBEdOgeLr3ZTOmn/F0QRxVfRF4MWDbvX6vlwJLg5zXiDMiKnFl5sGUq2DtEjj/fic5jweaW9vRphoG5eYzJoJF+EbkZsa0ZlF+sIFTxxfw5M29m7O7WyNnOOtlbf576GCx9Q2o+Agu/XXko6eMMTaDOyoz5jsZ7N5/yrNbvPThbrK0ngnFwyM6b0RuZkxrFuUH6ynOy4zZ9WIiORXGzXJWodUQ4xveXeAsb35cL/QxGdMPWbCIxtDjYPQZ8N5vob3Nk1v87p+bSJM2Rg6PrEO2KDeTnQcb0FBfomGobWrlYH1L3IbhdmvCeVBd7tQaunJwm5O34qQbIdWj0VnG9HMWLKI1c76zwODHL8f80uvKD/HJjt0AJEW4iGBRbiZ1zW1UN0bf1VN+sB6AkfkJVrMAJ1hA97O5Sx4DBKbf3CtFMqY/smARrU9dDAOLnFnBMfbEijIGpzU7byIMFrEcPlt+wLlGQtYsBhXD4Emw6bXg+1saYPUiOPZi51hjTI9YsIhWcorzG+uW16GiNGaX3V/bxF/X7mbuJDdIRBwsnOaWmAQLt2aRcH0WPhPPg+3/cuZRBPrgD04OEkubakxULFjEwkk3Okl0/GoXyzbupbqxpceXXLJyO81t7VwyyZ2rEOHy5L5Z3LEYEVV+sIGM1CQKstOivpYnJpwHbc1Q9lbn7arOZzLkOBh9enzKZkw/YcEiFrIL4fgrYM0z0FjFjgP13PJkCXcuXk1be+QdzC1t7fzune2cObGQ4iy34zzCmkXhgHTSkpNiFiyK845cViRhjDoVUrOP7LfY/o4zaXLGbTZc1pgoWbCIlRm3Ocl01jzDht3VALy1aT+/XBb5KiWvrt/LnupGbjh1jF9K1ciCRVKSMDw3g12HGiO+f6AdiThs1l9KOow9y+m38B/9tfI3zszuKVfFr2zG9BMWLGKl6EQoPhlWLuDj3VUAXDxlOL/8xyZeL40s98WTK8oYmZ/J2ccOiSql6ohBsZlrUX6wgZGJ2Lntb+J5cGibs6osQPUu2PACTPsCpIU/mdEYE5wFi1ia8UU48AlJW5czMj+TB688gU8NzeFrv18TdnPQhl3VrCw7wPWnjCE5SaILFjGYmFfd2EJVQ0ti1yzAbwitOyqq5HEnm97Jt8avTMb0IxYsYmnyXMgewkl7/8CnhuaQmZbM/37+RFralDsXrw4r1emTK8rITE0+nF2uqQaQHv12XJSbwd7qRlraep5idefBBB426y9vDBRMdPotWptg1eNwzL9B/th4l8yYfsGCRSylpNF64g3MaFnFzFynKWrc4AH8+MoprNlxiP9+cWO3px+sa+b5NTu5dFoRg7LcxDzNtU6togcdtCNyM2lX2FPV836L8o5gkeA1C3BqF2Vvw9pnoK7C0qYaE0MWLGJsy6iraCOJc6pf6Nh20aeHc8sZY3liRRl/Wbury3N/X7KDptZ2bjjNb3XZpupOKVUjUZQX/cS8hJ9j4W/iec5aXa/+PyiYAOPOjneJjOk3LFjE2IbabF5uP5kxO57rNEnsnjnHctLoPO754zo27zty8lhbu/LUv7Zxyrh8jh3mlweiqbZH/RXgN4u7qufBYseBBjJTk8lP1DkW/kafDikZbs6K+ZBk/7yNiRX73xRjH+2pYbH+G8nN1VB6eHX21OQkHr52GumpydyxeBX1zZ3XbPr7xr3sPNRwZM5qN/92T4wYdDi9ak+VH6xnZH5m4s6x8Jea6QyhTRsAJ8yLd2mM6VcsWMRY6Z5q6vOPd95UlXfaN3xQJr+4Ziqb9tXy7ec+7LQi7JMryhgxKIPzJgWsLttcG/HsbZ/MNKdGEM3EPN+EvD7j4p/CDX+BDI+z9BlzlLFgEWMf761lzPDBTnNIw8Ej9p85cTBfPfcYnnt/J0+7aUo/3lvDik8que7U0aQkB3wkUdQswFkjyjeiqScSMo9Fd3JHOXNejDExZcEihqobW9h5qIFPDctxMukFCRYA/37OBM46ZjD3vbCBD8qreHJFGWkpSVxz8qgjD26qiXj2tr+iKOZaVDW0UN3Y2reChTHGExYsYmjTXmcC3aeGdh8skpKEh66eSuGANG5fvIo/rd7J3BNGBO9Ejrpm4QSLniRB6jNzLIwxnvM0WIjIhSJSKiKbReSeIPtHi8gyEVknIq+LSLHfvh+LyHoR2Sgiv5Q+0MP60R43WHTULA51eWx+dhoPf/5E9lY30tDSxg2BHdvgrHMUZbDoSILUEHkSpB19adisMcZTngULEUkGHgHmAJOBeSIyOeCwB4FFqjoFuB/4oXvuacDpwBTgeOBkYJZXZY2Vj/fUkJ2W7CwP3k3NwufEUXn87Kqp3DF7PMcXDTrygNZG0LYed3DD4eGzPenk9k3IS/h1oYwxnvOyZjED2KyqW1S1GVgCzA04ZjKwzH293G+/AhlAGpAOpAJ7PSxrTHy0p4ZjhuWQlCSQmRsyWAB89oQRfP3CY4PvjGJdKJ/ogkU92WnJ5PpmkxtjjlpeBosiYIff+3J3m7+1wBXu68uAHBEpUNV/4QSP3e7PK6p6xFoZIjJfREpEpKSioiLmDxAJVaV0bw3HDnO/2DPzoOFAdBft4fLk/qLJmJfweSyMMb3Gy2AR7BsmsJf1bmCWiLyP08y0E2gVkQnAJKAYJ8CcIyJnHXEx1QWqOl1Vpw8ePDi2pY9QRU0Th+pbOGaoX7BobXRyQPdUDGoWhdnppKUkRREsrL/CGONtsCgHRvq9LwY6LYykqrtU9XJVnQZ8291WhVPLeEdVa1W1FngJOMXDskatU+c2QGa+82cYTVFd6ggWPe+zSEoSRgzK6HEzlAULYwx4GyzeAyaKyFgRSQOuAV7wP0BECkXEV4ZvAgvd19txahwpIpKKU+vofsnWOPvYf9gsODULiC5YNLtrSEVRs4Ce5bWoamihprHVhs0aYwAPg4WqtgJfBl7B+aJ/VlXXi8j9InKJe9hsoFREPgaGAg+425cCnwAf4PRrrFXVv3hV1lj4aE8NhQPSKRiQ7myIRbDoqFlEt3SFEywiWx9qxwFn2OzIfKtZGGMgxcuLq+qLwIsB2+71e70UJzAEntcGfNHLssVa6R6/zm2IUbBwcnn3dIlynxG5meytaaS5tZ20lPB+Pyi3CXnGGD82gzsG2tqVTftqDnduQ4yCRWyaoYpzM1GFvdXh1y76VB4LY4znLFjEwPYD9TS2tHtQs6gBSXKW3o5CT+ZalB9sYEB6CoMybY6FMcaCRUyUBo6EAidndlIq1Ecx1yKKlKr+ejLXwjds1uZYGGPAgkVMlO6pQQQmDvXrWxAJa8mPbjXVRN25DX4Z8yIKFjZs1hhzmAWLGCjdW82o/Cyy0gLGC2TlR9/BHWXnNkBGajIFESRBUtW+l/TIGOMpCxYxULonoHPbJ+qaRc/zbwcqystkZ5jDZ6saWqhtsjwWxpjDLFhEqbGljbLK+s6d2z4hlikPqakmqtnb/kYMCn9ing2bNcYEsmARpU8qamlr186d2z7R1iyaY1eziCQJkg2bNcYEsmARpY6RUJ40Q0WX+MjfiNwM6pvbqGpoCXms5bEwxgSyYBGl0r01pCUnMaYw+8idmbnQUgetTT27eJT5t/0VRTDXovxgAznpKQzM9HSCvzGmD7FgEaXSPTWMG5xNanKQv8poJubFIKWqv46JeQdDB4sdB+opzrc8FsaYwyxYROmINaH8RRMsmusAjVkHd1Fe+HMtLI+FMSaQBYuvCiwmAAAgAElEQVQoVDW0sLuqkU8N62LiXDQ5LWK0PLlPQXaakwSpqvvhs84cC5uQZ4zpzIJFFDpyWAzr4rf/aGoWMVqe3EdEKMrNDNlncai+hbrmNhs2a4zpxIJFFA6vCdVVzSKaYBGb5cn9jcjNCNkMdXiOhdUsjDGHWbCIQumeGnLSUxgxKCP4AVEFi9g2Q4EzMS9UB7fNsTDGBGPBIgqle2o4ZlhO16OG0nNAkqNshopdzaIoL5N9NU00tbZ1ecyOjmBhzVDGmMMsWPSQqlK6tyb4zG2faFaejXEHNxwePru3qut5H+UHGxiYYXksjDGdhQwWIpLcGwXpa/ZWN1HV0BJ85ra/zLye5bTw1SxiNCkPwpuYZ6vNGmOCCadmsVlEfiIikz0vTR9SujdIwqNgelqz8HVwe1Cz6K6T24bNGmOCCSdYTAE+Bn4rIu+IyHwRCWs8p4hcKCKlIrJZRO4Jsn+0iCwTkXUi8rqIFLvbzxaRNX4/jSJyaURP5rHSPc6XeciaRU9zWjTVOpn2UtJ7ULrghg/qPmOe5bEwxnQlZLBQ1RpVfVRVTwO+DnwX2C0iT4rIhK7Oc5uvHgHmAJOBeUFqJw8Ci1R1CnA/8EP3nstVdaqqTgXOAeqBVyN/PO98tKeGITnp5GWndX9gT5cp9y1PHsMlNzJSkykc0HUSpIP1LdQ3t1nNwhhzhLD6LETkEhF5DvgF8FNgHPAX4MVuTp0BbFbVLaraDCwB5gYcMxlY5r5eHmQ/wJXAS6paH6qsvenjUJ3bPtF0cMewCcqnu4l5vmGzI/OtZmGM6SycZqhNOF/iP1HVaar6M1Xdq6pLgZe7Oa8I2OH3vtzd5m8tcIX7+jIgR0QKAo65Bngm2A3cJrESESmpqKgI41Fio61d2bS3NnQTFDjBorkG2kIvDd5JDFec9efLaxHMjgM2Ic8YE1xYfRaqeouqrgjcoap3dXNesPaTwMw7dwOzROR9YBawE2jtuIDIcODTwCvBbqCqC1R1uqpOHzx4cIjHiJ1tlXU0tbaHX7OAyJuimqo9qVk4waIxaBIkX82iyIKFMSZAOAkLWkXkTuA4oGOqsqreHOK8cmCk3/tiYJf/Aaq6C7gcQEQGAFeoapXfIVcBz6lqhL+We+vwMh+RBIuDMCCCgNZUC1mBlazojcjNpKGljUP1LUf0t5QfbGBQZioDM2yOhTGms3BqFk8Bw4B/A97A+dKvCeO894CJIjJWRNJwmpNe8D9ARApFxFeGbwILA64xjy6aoOKpdG8NIjBxSDjBItf5syHCuRYxzL/tryjXiffB+i1s2KwxpivhBIsJqvr/gDpVfRK4GKdpqFuq2gp8GacJaSPwrKquF5H7ReQS97DZQKmIfAwMBR7wnS8iY3BqJm+E/TS9pHRPDWMKsslMC2O+Yk/Xh/Ksg9vpvA4eLCyPhTEmuHCaoXxNQIdE5HhgDzAmnIur6osEjJhS1Xv9Xi8FlnZxbhlHdognhNI9NRwzNMzf+nua08KzDu7gcy18cyxmHdN7fT/GmL4jnJrFAhHJA76D04y0AfgfT0uVwBpb2iirrOt6WfJAPalZtLd7VrPIz04jPSXpiGBRWddMQ4vNsTDGBNdtzcLtT6hW1YPAmzjzK45qm/fV0q5hzNz2SR8IkhRZsPBgEUEfXxKkXYc6Z8w7nMfC5lgYY47Ubc1CVdtx+h2MK6KRUABJSZCRG1mw8GB5cn8jgkzM68hjkW81C2PMkcJphnpNRO4WkZEiku/78bxkCap0bw1pKUmMKYjgN/BIZ3F7WLMAp9/iyGDhvPetTGuMMf7C6eD2zae402+bcpQ2SX20p4YJgweQkhxBKpBIg4UHy5P7K8rNosJNgpSe4ozoKj9YT25WKjk2x8IYE0TIYKGqY3ujIH3Fx3tqOG18hJPlMvOgLoLlSDqaobyrWQDsqWpkdEE24NQsRlp/hTGmCyGDhYhcH2y7qi6KfXESW1V9C3uqGzkm3P4Kn8w82P9x+Md7HCz8kyD5gsWOA/UcE26nvTHmqBNOM9TJfq8zgHOB1cBRFyzCTngUKCs/srWheqGDG+gYEeWbY3HOsUM8uZ8xpu8Lpxnq3/3fi8ggnCVAjjq+hEfH9qRm0VQFba2QHEZ87ujgDnMuR4SGuUmQdrqd2vtrm2lqbbdhs8aYLkXQS9uhHpgY64L0BaV7a8jJSGHYwIzQB/vzTcxrrOr+OB9fStU0b2oWGanJDM5J75iY1zFs1ibkGWO6EE6fxV84vLR4Ek7Come9LFSi2rq/jvGDByCRZq/zn8WdHUbneFMtJKdDSogsfFEYkZvJripfsLAJecaY7oXTZ/Gg3+tWYJuqlntUnoS2taKOmeN6sGx4pEt+NNV41rntU5SbwUfuBMPDwcJqFsaY4MIJFtuB3araCCAimSIyxl3o76jR2NLGrqpGxhZmR35yj4KFN01QPiMGZfKPj/ahquw4WE9+dhrZ6eH8czDGHI3C6bP4A9Du977N3XZUKausA4gyWISZ08KjRQT9jcjNpLGlnYP1LbY0uTEmpHCCRYqqNvveuK+9a0xPUFsrYhEsIqhZeDR728c3fHbnwQZLemSMCSmcYFHhl6wIEZkL7PeuSIlpq1uzGNOTYJExCJCE6rPwBYedh+rZebDBOreNMd0Kp5H6S8BiEXnYfV8OBJ3V3Z9trahjSE46A3rSrp+U7ASMSIJFwfjI7xMBX81ibXmVO8fCahbGmK6FMynvE+AUERkAiKqGk3+739m6v65nTVA+kSwm2As1i7ysVDJSk1i51elHsXWhjDHdCdkMJSL/LSK5qlqrqjUikiciP+iNwiWSXg0WvdDBLSKMyM1kXbmzDInVLIwx3Qmnz2KOqnYsbORmzbsonIuLyIUiUioim0XkniD7R4vIMhFZJyKvi0ix375RIvKqiGwUkQ0iMiace3qhqqGFyrrm3gkWba3QUu95Bzc4Cwq2tDnzLYssWBhjuhFOsEgWkXTfGxHJBNK7Od53XDLwCDAHZ9b3PBGZHHDYg8AiVZ0C3A/80G/fIuAnqjoJmAHsC6OsnijbH8VIKJ9wg4XHiY/8+VafLchOIyvN5lgYY7oWzjfE74BlIvK4+/4m4MkwzpsBbFbVLQAisgSYC2zwO2Yy8DX39XLgeffYyThDdl8DUNXaMO7nma1usBg3uBeChccrzvrzdXJbE5QxJpSQNQtV/THwA2ASzpf7y8DoMK5dBOzwe1/ubvO3FrjCfX0ZkCMiBcAxwCER+ZOIvC8iP3FrKp2IyHwRKRGRkoqKCJILRWjL/jqSBEbmR9EJnJnnLFPe3tb9cR7nsvB3OFhY57Yxpnvhrjq7B2cW9xU4+Sw2hnFOsNX2NOD93cAsEXkfmAXsxFl/KgU4091/Mk4K1xuPuJjqAlWdrqrTBw8eHN6T9EDZ/jqK8jI7UpD2SFY+oKFXnu3FZihfxrzifKtZGGO612UzlIgcA1wDzAMqgd/jDJ09O8xrlwMj/d4XA7v8D1DVXcDl7v0GAFeoapWIlAPv+zVhPQ+cAjwW5r1jyhkJFWWzkP8s7qz8ro/rWJ7c+2Axyq0pjSmIonnNGHNU6K5m8RFOLeKzqnqGqv4KZ12ocL0HTBSRsSKShhN4XvA/QEQKRcRXhm8CC/3OzRMRX3XhHDr3dfQaVXWCRUGUTTUdwSJExrym3qtZFOdl8fRtM7lsWmDroDHGdNZdsLgCp/lpuYg8KiLnErxpKShVbQW+DLyC02z1rKquF5H7/ZYPmQ2UisjHwFDgAffcNpwmqGUi8oF730cjerIY2V/bTG1Ta3QjoSD89aF6sYMb4LTxhWSkRtG8Zow5KnTZDKWqzwHPiUg2cCnOqKWhIvJr4DlVfTXUxVX1ReDFgG33+r1eCizt4tzXgCnhPISXfCOhxg6OYTNUd3qxg9sYY8IVzmioOlVdrKqfwel3WAMcMcGuv9q632kWGtdbNQtfB3cv9FkYY0y4IsrBraoHVPU3qnqOVwVKNFv215GWnNQxzLTHMnKdP0PWLKohJROSbZKcMSZxRBQsjkZl++sYVZBFclKEebcDJadA+sDQCZCavF8XyhhjImXBIoSoFxD0F84s7l5IqWqMMZGyYNGNtnalrLI+DsHCahbGmMRiwaIbuw410Nza3rvBornWaa4yxpgEYsGiG2WVMVht1l9YNYtqSLNmKGNMYrFg0Y2O1WZ7NVhYB7cxJvFYsOjGloo6stOSGZwTMn1HeHzBor2962Osg9sYk4AsWHSjrLKOMYXZiEQ5bNYnMw+0HZq7SWNuHdzGmARkwaIbMR02C4dncdd3MdeitRnammz2tjEm4Viw6EJzazs7DtTHrr8CDi9N3lW/RS/msjDGmEhYsOjC9gP1tCuM8aJm0VWwsEUEjTEJyoJFF8r2x3jYLEQQLKyD2xiTWCxYdGFrPIKFNUMZYxKUBYsubNlfR15WKrlZabG7aMfKs11ky/PVLKyD2xiTYCxYdGHr/trY1ioAUtKc2dldNkO5+betZmGMSTAWLLpQtr+esYUe9B10N4u7F/NvG2NMJCxYBFHX1Mqe6kbGDY5xzQLcYNHFPAvr4DbGJCgLFkH4FhAcU+BVsAjRwW0LCRpjEoynwUJELhSRUhHZLCJH5O0WkdEiskxE1onI6yJS7LevTUTWuD8veFnOQGX764EYj4Ty6bYZqgZSsyEpOfb3NcaYKHiW6FlEkoFHgPOBcuA9EXlBVTf4HfYgsEhVnxSRc4AfAl9w9zWo6lSvytedrfud3/DHFGbF/uLdBotq668wxiQkL2sWM4DNqrpFVZuBJcDcgGMmA8vc18uD7I+LLfvrGDYwg6w0D2KpL1ioHrnPlic3xiQoL4NFEbDD7325u83fWuAK9/VlQI6IFLjvM0SkRETeEZFLg91AROa7x5RUVFTErOAxX0DQX2YetLce7p/wZ8uTG2MSlJfBIti63oG/Tt8NzBKR94FZwE6g1d03SlWnA9cCD4nI+CMuprpAVaer6vTBgwfHrOBl++sY68VIKOh+Fnez1SyMMYnJy2BRDoz0e18M7PI/QFV3qerlqjoN+La7rcq3z/1zC/A6MM3DsnY4WNfMwfqW2K4266+7YNFUY7O3jTEJyctg8R4wUUTGikgacA3QaVSTiBSKiK8M3wQWutvzRCTddwxwOuDfMe6ZrV4Om4XDy5QHy2lhHdzGmATlWbBQ1Vbgy8ArwEbgWVVdLyL3i8gl7mGzgVIR+RgYCjzgbp8ElIjIWpyO7x8FjKLyzNYKdwHBeDRDWQe3MSZBeTZ0FkBVXwReDNh2r9/rpcDSIOetAD7tZdm6UlZZR3KSMDLPg2GzELoZyjq4jTEJyGZwB9iyv46ReZmkpXj0V9Ox8mxAsGhtgvYWq1kYYxKSBYsAWyvqYpsdL1BqBqRmHRksbHlyY0wCs2DhR1Upq/RwjoVPZt6ROS1seXJjTAKzYOFnX00T9c1t3g2b9Qm25EfH8uTWZ2GMSTwWLPxscUdCedoMBV0EC9/y5FazMMYkHgsWfjzJux1MsJwWln/bGJPALFj4KausIy0liRGDMr29UXc1C+vgNsYkIAsWfrZU1DG2IJukpGDLWsVQsJVnrYPbGJPALFj42bq/1pscFoEy86CtGVrqD2+zDm5jTAKzYOFqbWtn+4F6xhb2wpd1sFncTTWAOJnyjDEmwViwcO061EhLm3o/bBaCBwvf8uRJ9pEYYxKPfTO5tripVD1bQNBf0JpFNaRZE5QxJjFZsHD5hs16tjS5v66aoaxz2xiToCxYuMr215GTnkLhgDTvbxYsp0VTrXVuG2MSlgUL1xY3laqIx8NmwWoWxpg+x4KFa+v+ut5pggJIzYSUjOAd3MYYk4AsWACNLW3sPNTg/TIf/gJncVv+bWNMArNgAew4UI8qjOuNkVA+RwQLy79tjElcFixw+iugFxYQ9Oef00LVOriNMQnN02AhIheKSKmIbBaRe4LsHy0iy0RknYi8LiLFAfsHishOEXnYy3J2DJuNVzNUSwNom9UsjDEJy7NgISLJwCPAHGAyME9EJgcc9iCwSFWnAPcDPwzY/33gDa/K6LO1oo7CAWkMzEj1+laHZeYeDha2PLkxJsGleHjtGcBmVd0CICJLgLnABr9jJgNfc18vB5737RCRk4ChwMvAdA/LydbeSKUaKDP/cE4LW57cmE5aWlooLy+nsbEx3kXpNzIyMiguLiY1tWe/FHsZLIqAHX7vy4GZAcesBa4AfgFcBuSISAFwEPgp8AXg3K5uICLzgfkAo0aN6nFBt+6vY/Yxg3t8fo9k5kFro9MEZVnyjOmkvLycnJwcxowZ0ztzn/o5VaWyspLy8nLGjh3bo2t42WcR7BPWgPd3A7NE5H1gFrATaAXuAF5U1R10Q1UXqOp0VZ0+eHDPvuxrGluoqGnqnTWh/PlPzOsIFtbBbQxAY2MjBQUFFihiREQoKCiIqqbmZc2iHBjp974Y2OV/gKruAi4HEJEBwBWqWiUipwJnisgdwAAgTURqVfWITvJotbUrd8wezynjCmJ96e4FDRZWszDGxwJFbEX79+llsHgPmCgiY3FqDNcA1/ofICKFwAFVbQe+CSwEUNXP+x1zIzDdi0ABkJuVxtcvPNaLS3fPP1j4Oritz8IYk6A8a4ZS1Vbgy8ArwEbgWVVdLyL3i8gl7mGzgVIR+RinM/sBr8qTcDrVLCylqjGJpLKykqlTpzJ16lSGDRtGUVFRx/vm5uawrnHTTTdRWlrqcUl7j5c1C1T1ReDFgG33+r1eCiwNcY0ngCc8KF58dQoWNnTWmERSUFDAmjVrAPje977HgAEDuPvuuzsdo6qoKkldJCx7/PHHPS9nb/I0WJhuBPZZSJKzwKAxppP7/rKeDbuqY3rNySMG8t3PHhfxeZs3b+bSSy/ljDPO4N133+Wvf/0r9913H6tXr6ahoYGrr76ae+91fh8+44wzePjhhzn++OMpLCzkS1/6Ei+99BJZWVn8+c9/ZsiQITF9Jq/Zch/xkpYNyWlOTgvf8uTWoWdMwtuwYQO33HIL77//PkVFRfzoRz+ipKSEtWvX8tprr7Fhw4YjzqmqqmLWrFmsXbuWU089lYULF8ah5NGxmkW8iBxe8qO91Tq3jelCT2oAXho/fjwnn3xyx/tnnnmGxx57jNbWVnbt2sWGDRuYPLnzYhWZmZnMmTMHgJNOOom33nqrV8scCxYs4qljfSi1/gpj+ojs7MNzsjZt2sQvfvELVq5cSW5uLtddd13QuQxpaYczcCYnJ9Pa2torZY0la4aKJ1+waLLER8b0RdXV1eTk5DBw4EB2797NK6+8Eu8iecZqFvGUmQeHdkBKOmQMjHdpjDEROvHEE5k8eTLHH38848aN4/TTT493kTwjqoErcPRN06dP15KSkngXIzLP3wFb3nA6u4ccC1ctineJjEkIGzduZNKkSfEuRr8T7O9VRFapasjFWq1mEU/+fRbWwW2MSWAWLOIpMxda6izxkTEm4VkHdzxl5jt/tjZasDDGJDQLFvHkm8UNtjy5MSahWbCIp07BwmoWxpjEZcEinvyDhXVwG2MSmAWLeLKahTEJafbs2UdMsHvooYe44447ujxnwACnKXnXrl1ceeWVXV431BD/hx56iPr6+o73F110EYcOHQq36J6xYBFPFiyMSUjz5s1jyZIlnbYtWbKEefPmhTx3xIgRLF3abeaFbgUGixdffJHc3NweXy9WbOhsPKXngCS7Q2etg9uYoF66B/Z8ENtrDvs0zPlRl7uvvPJKvvOd79DU1ER6ejplZWXs2rWLqVOncu6553Lw4EFaWlr4wQ9+wNy5czudW1ZWxmc+8xk+/PBDGhoauOmmm9iwYQOTJk2ioaGh47jbb7+d9957j4aGBq688kruu+8+fvnLX7Jr1y7OPvtsCgsLWb58OWPGjKGkpITCwkJ+9rOfdaxYe+utt/LVr36VsrIy5syZwxlnnMGKFSsoKiriz3/+M5mZsU15YDWLePKtPAtWszAmgRQUFDBjxgxefvllwKlVXH311WRmZvLcc8+xevVqli9fzn/+53/S3SoYv/71r8nKymLdunV8+9vfZtWqVR37HnjgAUpKSli3bh1vvPEG69at46677mLEiBEsX76c5cuXd7rWqlWrePzxx3n33Xd55513ePTRR3n//fcBZ0HDO++8k/Xr15Obm8sf//jHmP+dWM0i3rLyoX6/dXAb05VuagBe8jVFzZ07lyVLlrBw4UJUlW9961u8+eabJCUlsXPnTvbu3cuwYcOCXuPNN9/krrvuAmDKlClMmTKlY9+zzz7LggULaG1tZffu3WzYsKHT/kBvv/02l112Wceqt5dffjlvvfUWl1xyCWPHjmXq1KmAswR6WVlZjP4WDrOaRbxZzcKYhHTppZeybNmyjix4J554IosXL6aiooJVq1axZs0ahg4dGnRJcn8SJKnZ1q1befDBB1m2bBnr1q3j4osvDnmd7mow6enpHa+9WgLdgkW8ZeZBUoqz8qwxJmEMGDCA2bNnc/PNN3d0bFdVVTFkyBBSU1NZvnw527Zt6/YaZ511FosXLwbgww8/ZN26dYCztHl2djaDBg1i7969vPTSSx3n5OTkUFNTE/Razz//PPX19dTV1fHcc89x5plnxupxQ/I0WIjIhSJSKiKbReSeIPtHi8gyEVknIq+LSLHf9lUiskZE1ovIl7wsZ1xl5kHaAEupakwCmjdvHmvXruWaa64B4POf/zwlJSVMnz6dxYsXc+yxx3Z7/u23305tbS1Tpkzhxz/+MTNmzADghBNOYNq0aRx33HHcfPPNnZY2nz9/PnPmzOHss8/udK0TTzyRG2+8kRkzZjBz5kxuvfVWpk2bFuMn7ppnS5SLSDLwMXA+UA68B8xT1Q1+x/wB+KuqPiki5wA3qeoXRCTNLVuTiAwAPgROU9VdXd2vTy5RDrD9Xdi3HqbfHO+SGJMwbIlyb0SzRLmXNYsZwGZV3aKqzcASYG7AMZOBZe7r5b79qtqsqk3u9nSPyxlfo2ZaoDDGJDwvv4SLgB1+78vdbf7WAle4ry8DckSkAEBERorIOvca/xOsViEi80WkRERKKioqYv4AxhhjHF4Gi2CN8IFtXncDs0TkfWAWsBNoBVDVHao6BZgA3CAiQ4+4mOoCVZ2uqtMHDx4c29IbY+Kqv2TxTBTR/n16GSzKgZF+74uBTrUDVd2lqper6jTg2+62qsBjgPVA73X7G2PiKiMjg8rKSgsYMaKqVFZWkpGR0eNreDkp7z1gooiMxakxXANc63+AiBQCB1S1HfgmsNDdXgxUqmqDiOQBpwM/87CsxpgEUlxcTHl5Oda8HDsZGRkUFxf3+HzPgoWqtorIl4FXgGRgoaquF5H7gRJVfQGYDfxQRBR4E7jTPX0S8FN3uwAPqmqMF4cxxiSq1NRUxo4dG+9iGD+eDZ3tbX126KwxxsRRIgydNcYY009YsDDGGBNSv2mGEpEKoPuFWrpXCOyPUXESQX97Huh/z9Tfngf63zP1t+eBI59ptKqGnHvQb4JFtESkJJx2u76ivz0P9L9n6m/PA/3vmfrb80DPn8maoYwxxoRkwcIYY0xIFiwOWxDvAsRYf3se6H/P1N+eB/rfM/W354EePpP1WRhjjAnJahbGGGNCsmBhjDEmpKM+WIRK/doXiUiZiHzgpqXtc2ugiMhCEdknIh/6bcsXkddEZJP7Z148yxipLp7peyKy0/2c1ojIRfEsYyTcfDPLRWSjm/r4K+72Pvk5dfM8ffkzyhCRlSKy1n2m+9ztY0XkXfcz+r2bmTT09Y7mPotwUr/2RSJSBkxX1T45mUhEzgJqgUWqery77cc4KxT/yA3qear6jXiWMxJdPNP3gFpVfTCeZesJERkODFfV1SKSA6wCLgVupA9+Tt08z1X03c9IgGxVrRWRVOBt4CvAfwB/UtUlIvJ/wFpV/XWo6x3tNYtwUr+aXqaqbwIHAjbPBZ50Xz+J8x+5z+jimfosVd2tqqvd1zXARpxMmH3yc+rmefosddS6b1PdHwXOAZa628P+jI72YBFO6te+SIFXRWSViMyPd2FiZKiq7gbnPzYwJM7liZUvi8g6t5mqTzTZBBKRMcA04F36wecU8DzQhz8jEUkWkTXAPuA14BPgkKq2uoeE/Z13tAeLcFK/9kWnq+qJwBzgTrcJxCSeXwPjganAbuCn8S1O5ERkAPBH4KuqWh3v8kQryPP06c9IVdtUdSpOptIZOLmCjjgsnGsd7cEiZOrXvshNRYuq7gOew/lH0tftdduVfe3L++Jcnqip6l73P3M78Ch97HNy28H/CCxW1T+5m/vs5xTsefr6Z+SjqoeA14FTgFwR8SW+C/s772gPFh2pX90RAdcAL8S5TFERkWy3gw4RyQYuAD7s/qw+4QXgBvf1DcCf41iWmPB9qbouow99Tm7n6WPARlX1T3ncJz+nrp6nj39Gg0Uk132dCZyH0xezHLjSPSzsz+ioHg0F4A6Fe4jDqV8fiHORoiIi43BqE+CkzX26rz2TiDyDk3K3ENgLfBd4HngWGAVsBz6nqn2mw7iLZ5qN07yhQBnwRV97f6ITkTOAt4APgHZ387dw2vn73OfUzfPMo+9+RlNwOrCTcSoGz6rq/e53xBIgH3gfuE5Vm0Je72gPFsYYY0I72puhjDHGhMGChTHGmJAsWBhjjAnJgoUxxpiQLFgYY4wJyYKFMREQkTa/FUjXxHKlYhEZ478qrTGJJCX0IcYYPw3u8gnGHFWsZmFMDLg5RP7HzR+wUkQmuNtHi8gydyG6ZSIyyt0+VESec3MNrBWR09xLJYvIo27+gVfdmbfGxJ0FC2MikxnQDHW1375qVZ0BPIyzKgDu60WqOgVYDPzS3f5L4A1VPQE4EVjvbp8IPKKqxwGHgCs8fh5jwmIzuI2JgIjUquqAINvLgHNUdYu7ILZ9reEAAADhSURBVN0eVS0Qkf04SXVa3O27VbVQRCqAYv9lFtylsV9T1Ynu+28Aqar6A++fzJjuWc3CmNjRLl53dUww/mv0tGH9iiZBWLAwJnau9vvzX+7rFTirGQN8Hie1JcAy4HboSFAzsLcKaUxP2G8txkQm08085vOyqvqGz6aLyLs4v4TNc7fdBSwUkf8CKoCb3O1fARaIyC04NYjbcZLrGJOQrM/CmBhw+yymq+r+eJfFGC9YM5QxxpiQrGZhjDEmJKtZGGOMCcmChTHGmJAsWBhjjAnJgoUxxpiQLFgYY4wJ6f8DfJdGXlYqGY8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_history(history[best_index][\"fit_history\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      1740\n",
      "           1       1.00      1.00      1.00       344\n",
      "\n",
      "   micro avg       1.00      1.00      1.00      2084\n",
      "   macro avg       1.00      1.00      1.00      2084\n",
      "weighted avg       1.00      1.00      1.00      2084\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model(history[best_index][\"model\"], train_sequences, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00       192\n",
      "           1       1.00      1.00      1.00        39\n",
      "\n",
      "   micro avg       1.00      1.00      1.00       231\n",
      "   macro avg       1.00      1.00      1.00       231\n",
      "weighted avg       1.00      1.00      1.00       231\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model(history[best_index][\"model\"], valid_sequences, valid_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.99      0.99       480\n",
      "           1       0.97      0.98      0.97        98\n",
      "\n",
      "   micro avg       0.99      0.99      0.99       578\n",
      "   macro avg       0.98      0.99      0.98       578\n",
      "weighted avg       0.99      0.99      0.99       578\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model(history[best_index][\"model\"], test_sequences, test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "for entry in history:\n",
    "    clean_model(entry[\"model\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>BiLSTM on Pretrained (Glove) Embeddings + Attention (MLP)</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "In (InputLayer)                 (None, 150)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "Embeddings (Embedding)          (None, 150, 100)     4745000     In[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "BiLSTM_Stack (Sequential)       (None, 150, 200)     403200      Embeddings[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "Attention (Sequential)          (None, 150, 1)       201         BiLSTM_Stack[1][0]               \n",
      "__________________________________________________________________________________________________\n",
      "Multiply (Multiply)             (None, 150, 200)     0           BiLSTM_Stack[1][0]               \n",
      "                                                                 Attention[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "Out (Sequential)                (None, 1)            201         Multiply[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 5,148,602\n",
      "Trainable params: 403,602\n",
      "Non-trainable params: 4,745,000\n",
      "__________________________________________________________________________________________________\n",
      "Train on 2084 samples, validate on 231 samples\n",
      "Epoch 1/30\n",
      "2084/2084 [==============================] - 6s 3ms/step - loss: 0.1705 - acc: 0.9395 - val_loss: 0.0435 - val_acc: 0.9870\n",
      "Epoch 2/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0691 - acc: 0.9798 - val_loss: 0.0441 - val_acc: 0.9957\n",
      "Epoch 3/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0623 - acc: 0.9770 - val_loss: 0.0082 - val_acc: 1.0000\n",
      "Epoch 4/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0296 - acc: 0.9894 - val_loss: 0.0149 - val_acc: 1.0000\n",
      "Epoch 5/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0226 - acc: 0.9918 - val_loss: 0.0602 - val_acc: 0.9610\n",
      "Epoch 6/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0167 - acc: 0.9952 - val_loss: 0.0036 - val_acc: 1.0000\n",
      "Epoch 7/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0145 - acc: 0.9947 - val_loss: 0.0019 - val_acc: 1.0000\n",
      "Epoch 8/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0088 - acc: 0.9976 - val_loss: 0.0614 - val_acc: 0.9740\n",
      "Epoch 9/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0052 - acc: 0.9981 - val_loss: 0.0015 - val_acc: 1.0000\n",
      "Epoch 10/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0195 - acc: 0.9962 - val_loss: 0.0021 - val_acc: 1.0000\n",
      "Epoch 11/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0110 - acc: 0.9947 - val_loss: 0.0028 - val_acc: 1.0000\n",
      "Epoch 12/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 0.0069 - acc: 0.9976 - val_loss: 0.0072 - val_acc: 0.9957\n",
      "Epoch 13/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 8.0691e-04 - acc: 1.0000 - val_loss: 3.3083e-04 - val_acc: 1.0000\n",
      "Epoch 14/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 2.2631e-04 - acc: 1.0000 - val_loss: 1.7611e-04 - val_acc: 1.0000\n",
      "Epoch 15/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 1.2976e-04 - acc: 1.0000 - val_loss: 1.1777e-04 - val_acc: 1.0000\n",
      "Epoch 16/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 8.9482e-05 - acc: 1.0000 - val_loss: 9.5948e-05 - val_acc: 1.0000\n",
      "Epoch 17/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 6.4582e-05 - acc: 1.0000 - val_loss: 7.2854e-05 - val_acc: 1.0000\n",
      "Epoch 18/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 4.9528e-05 - acc: 1.0000 - val_loss: 6.1232e-05 - val_acc: 1.0000\n",
      "Epoch 19/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 3.9731e-05 - acc: 1.0000 - val_loss: 5.0820e-05 - val_acc: 1.0000\n",
      "Epoch 20/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 3.2519e-05 - acc: 1.0000 - val_loss: 4.4199e-05 - val_acc: 1.0000\n",
      "Epoch 21/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 2.7175e-05 - acc: 1.0000 - val_loss: 3.6861e-05 - val_acc: 1.0000\n",
      "Epoch 22/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 2.2955e-05 - acc: 1.0000 - val_loss: 3.3048e-05 - val_acc: 1.0000\n",
      "Epoch 23/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 1.9689e-05 - acc: 1.0000 - val_loss: 2.9272e-05 - val_acc: 1.0000\n",
      "Epoch 24/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 1.7013e-05 - acc: 1.0000 - val_loss: 2.5990e-05 - val_acc: 1.0000\n",
      "Epoch 25/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 1.4889e-05 - acc: 1.0000 - val_loss: 2.2877e-05 - val_acc: 1.0000\n",
      "Epoch 26/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 1.3154e-05 - acc: 1.0000 - val_loss: 2.0322e-05 - val_acc: 1.0000\n",
      "Epoch 27/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 1.1608e-05 - acc: 1.0000 - val_loss: 1.8423e-05 - val_acc: 1.0000\n",
      "Epoch 28/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 1.0324e-05 - acc: 1.0000 - val_loss: 1.6658e-05 - val_acc: 1.0000\n",
      "Epoch 29/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 9.1899e-06 - acc: 1.0000 - val_loss: 1.4910e-05 - val_acc: 1.0000\n",
      "Epoch 30/30\n",
      "2084/2084 [==============================] - 4s 2ms/step - loss: 8.2341e-06 - acc: 1.0000 - val_loss: 1.3433e-05 - val_acc: 1.0000\n",
      "WARNING:tensorflow:From C:\\Users\\josep\\Anaconda3\\envs\\NLP\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0512 22:58:23.413309 13204 deprecation.py:506] From C:\\Users\\josep\\Anaconda3\\envs\\NLP\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "In (InputLayer)                 (None, 150)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "Embeddings (Embedding)          (None, 150, 100)     4745000     In[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "BiLSTM_Stack (Sequential)       (None, 150, 200)     403200      Embeddings[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "Attention (Sequential)          (None, 150, 1)       168705      BiLSTM_Stack[1][0]               \n",
      "__________________________________________________________________________________________________\n",
      "Multiply (Multiply)             (None, 150, 200)     0           BiLSTM_Stack[1][0]               \n",
      "                                                                 Attention[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "Out (Sequential)                (None, 1)            201         Multiply[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 5,317,106\n",
      "Trainable params: 572,106\n",
      "Non-trainable params: 4,745,000\n",
      "__________________________________________________________________________________________________\n",
      "Train on 2084 samples, validate on 231 samples\n",
      "Epoch 1/30\n",
      "2084/2084 [==============================] - 7s 3ms/step - loss: 0.1794 - acc: 0.9376 - val_loss: 0.0330 - val_acc: 0.9913\n",
      "Epoch 2/30\n",
      "2084/2084 [==============================] - 5s 2ms/step - loss: 0.0779 - acc: 0.9755 - val_loss: 0.0125 - val_acc: 1.0000\n",
      "Epoch 3/30\n",
      "2084/2084 [==============================] - 5s 2ms/step - loss: 0.0544 - acc: 0.9794 - val_loss: 0.0162 - val_acc: 1.0000\n",
      "Epoch 4/30\n",
      "2084/2084 [==============================] - 5s 2ms/step - loss: 0.0353 - acc: 0.9870 - val_loss: 0.0208 - val_acc: 0.9957\n",
      "Epoch 5/30\n",
      "2084/2084 [==============================] - 5s 2ms/step - loss: 0.0329 - acc: 0.9885 - val_loss: 0.0162 - val_acc: 1.0000\n",
      "Epoch 6/30\n",
      "2084/2084 [==============================] - 5s 2ms/step - loss: 0.0254 - acc: 0.9909 - val_loss: 0.0075 - val_acc: 1.0000\n",
      "Epoch 7/30\n",
      "2084/2084 [==============================] - 5s 2ms/step - loss: 0.0193 - acc: 0.9923 - val_loss: 0.0158 - val_acc: 0.9913\n",
      "Epoch 8/30\n",
      "2084/2084 [==============================] - 5s 2ms/step - loss: 0.0499 - acc: 0.9832 - val_loss: 0.0158 - val_acc: 0.9913\n",
      "Epoch 9/30\n",
      "2084/2084 [==============================] - 5s 2ms/step - loss: 0.0184 - acc: 0.9923 - val_loss: 0.0027 - val_acc: 1.0000\n",
      "Epoch 10/30\n",
      "2084/2084 [==============================] - 5s 2ms/step - loss: 0.0041 - acc: 0.9986 - val_loss: 0.0931 - val_acc: 0.9697\n",
      "Epoch 11/30\n",
      "2084/2084 [==============================] - 5s 2ms/step - loss: 0.0115 - acc: 0.9966 - val_loss: 0.0047 - val_acc: 0.9957\n",
      "Epoch 12/30\n",
      "2084/2084 [==============================] - 5s 2ms/step - loss: 0.0041 - acc: 0.9986 - val_loss: 0.0021 - val_acc: 1.0000\n",
      "Epoch 13/30\n",
      "2084/2084 [==============================] - 5s 2ms/step - loss: 0.0016 - acc: 0.9995 - val_loss: 6.1463e-04 - val_acc: 1.0000\n",
      "Epoch 14/30\n",
      "2084/2084 [==============================] - 5s 2ms/step - loss: 0.0014 - acc: 0.9990 - val_loss: 3.8589e-04 - val_acc: 1.0000\n",
      "Epoch 15/30\n",
      "2084/2084 [==============================] - 5s 2ms/step - loss: 9.1466e-04 - acc: 0.9995 - val_loss: 2.5607e-04 - val_acc: 1.0000\n",
      "Epoch 16/30\n",
      "2084/2084 [==============================] - 5s 2ms/step - loss: 5.4800e-04 - acc: 0.9995 - val_loss: 2.0268e-04 - val_acc: 1.0000\n",
      "Epoch 17/30\n",
      "2084/2084 [==============================] - 5s 2ms/step - loss: 0.0259 - acc: 0.9899 - val_loss: 0.0595 - val_acc: 0.9827\n",
      "Epoch 18/30\n",
      "2084/2084 [==============================] - 5s 2ms/step - loss: 0.0366 - acc: 0.9875 - val_loss: 0.0141 - val_acc: 0.9957\n",
      "Epoch 19/30\n",
      "2084/2084 [==============================] - 5s 2ms/step - loss: 0.0155 - acc: 0.9942 - val_loss: 0.0122 - val_acc: 0.9913\n",
      "Epoch 20/30\n",
      "2084/2084 [==============================] - 5s 2ms/step - loss: 0.0264 - acc: 0.9933 - val_loss: 0.0338 - val_acc: 0.9870\n",
      "Epoch 21/30\n",
      "2084/2084 [==============================] - 5s 2ms/step - loss: 0.0108 - acc: 0.9962 - val_loss: 0.0056 - val_acc: 1.0000\n",
      "Epoch 22/30\n",
      "2084/2084 [==============================] - 5s 2ms/step - loss: 0.0022 - acc: 0.9986 - val_loss: 0.0172 - val_acc: 0.9913\n",
      "Restoring model weights from the end of the best epoch\n",
      "Epoch 00022: early stopping\n"
     ]
    }
   ],
   "source": [
    "param_grid = {\n",
    "    \"use_atention\": [True],\n",
    "    \"rnn_stack_size\": [2],\n",
    "    \"mlp_model\": [\n",
    "        Sequential([\n",
    "            TimeDistributed(Dense(1))\n",
    "        ]),\n",
    "        Sequential([\n",
    "            TimeDistributed(Dense(512, activation=\"relu\")),\n",
    "            Dropout(0.5),\n",
    "            TimeDistributed(Dense(128, activation=\"relu\")),\n",
    "            Dropout(0.5),\n",
    "            TimeDistributed(Dense(1))\n",
    "        ])\n",
    "    ]\n",
    "}\n",
    "\n",
    "history, best_index = tune_model(create_rnn_model, param_grid=param_grid, epochs=30, patience=6, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tune Results: #1, {'use_atention': True, 'rnn_stack_size': 2, 'mlp_model': <keras.engine.sequential.Sequential object at 0x000002241A8A46D8>}\n"
     ]
    }
   ],
   "source": [
    "print(\"Tune Results: #{}, {}\".format(best_index + 1, dict(zip(param_grid.keys(), history[best_index][\"params\"]))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl4VOXZ+PHvnX2FkIRFCLuooCIogrihtraiVlxwoa27pVqtdrHvq12s+uprf61trdXaQotLXwq1WJe2uJWCG6iEVQFRNiEsmh2YJGS7f3+cM2EYJpmZZLZk7s915WLmnGfOeYaBufNs9yOqijHGGNORlHhXwBhjTOKzYGGMMSYoCxbGGGOCsmBhjDEmKAsWxhhjgrJgYYwxJigLFiapicgwEVERSQuh7HUi8nYs6mVMorFgYboNEdkmIo0iUux3fLX7hT8sPjU7pC65IrJfRBbGuy7GRJIFC9PdbAVmeJ+IyPFAdvyqc5jpwAHgSyJyRCxvHErryJjOsmBhups/A9f4PL8WeMa3gIj0FpFnRKRcRD4VkR+LSIp7LlVEHhaRChHZAlwQ4LV/EpHdIrJTRB4QkdQw6nct8HtgLfA1v2sPFpG/u/WqFJHHfM59Q0Q2iMg+EVkvIie6x1VEjvQp95SIPOA+PktEykTkv0VkD/CkiPQRkX+696h2H5f4vL5QRJ4UkV3u+Rfc4x+KyFd8yqW7f0fjwnjvpgezYGG6m3eBXiIy2v0SvxL4P78yvwV6AyOAKTjB5Xr33DeAC4HxwAScloCvp4Fm4Ei3zJeAm0KpmIgMAc4C5ro/1/icSwX+CXwKDAMGAfPdc5cD97rlewEXAZWh3BMYABQCQ4GZOP+nn3SfDwHqgcd8yv8ZyAGOBfoBv3aPPwN83afc+cBuVV0dYj1MT6eq9mM/3eIH2AZ8Efgx8BBwHvA6kAYozpdwKk430Bif130TWOI+/g9ws8+5L7mvTQP6u6/N9jk/A1jsPr4OeLuD+v0YWO0+Hgi0AOPd55OBciAtwOteBe5o55oKHOnz/CngAffxWUAjkNVBncYB1e7jI4BWoE+AcgOBfUAv9/kC4L/i/ZnbT+L8WB+n6Y7+DLwJDMevCwooBjJwfoP3+hTnN3lwvhR3+J3zGgqkA7tFxHssxa98R64BZgOo6i4ReQOnW2oVMBj4VFWbA7xuMLA5xHv4K1fVBu8TEcnBaS2cB/RxD+e7LZvBQJWqVvtfxK3vO8BlIvI8MBW4o5N1Mj2QdUOZbkdVP8UZ6D4f+Lvf6QqgCeeL32sIsNN9vBvnS9P3nNcOnJZFsaoWuD+9VPXYYHUSkVOBUcDdIrLHHUOYBMxwB553AEPaGYTeAYxs59J1ON1GXgP8zvunjf4+cDQwSVV7AWd6q+jep1BECtq519M4XVGXA8tUdWc75UwSsmBhuqsbgXNU1eN7UFVbgGeBB0UkX0SGAt/j4LjGs8DtIlIiIn2Au3xeuxt4DfiliPQSkRQRGSkiU0Koz7U4XWJjcLp+xgHH4XzRTwXexwlUP3On12aJyGnua/8I3CkiJ4njSLfeAKuBr7oD8+fhjMF0JB9nnKJGRAqBn/q9v5eB37kD4ekicqbPa18ATsRpUfi32EySs2BhuiVV3ayqpe2c/jbgAbYAbwN/Aea452bjjBGsAVZyeMvkGpxurPVANU7ffYdTYEUkC7gC+K2q7vH52YrTZXatG8S+gjNwvh0owxmcR1X/Bjzo1nMfzpd2oXv5O9zX1eDMrnqho7oAj+BMJa7AmQzwit/5q3FaXh8BnwPf8Z5Q1XrgOZzuPf+/F5PkRNU2PzLGOETkHuAoVf160MImqdgAtzEGcNZg4HTvXR3vupjEY91QxhhE5Bs4A+Avq+qb8a6PSTzWDWWMMSYoa1kYY4wJqseMWRQXF+uwYcPiXQ1jjOlWVqxYUaGqfYOV6zHBYtiwYZSWtjeT0hhjTCAi8mnwUtYNZYwxJgQWLIwxxgRlwcIYY0xQFiyMMcYEZcHCGGNMUFELFiIyR0Q+F5EP2zkvIvKoiGwSkbXebSTdc9eKyCfuz7XRqqMxxpjQRLNl8RTOBiztmYqT/38UznaQT0Bbfpqf4uwFMBH4qZtK2hhjTJxEbZ2Fqr4pIsM6KDINeEadfCPvikiBiByBs1Xk66paBSAir+MEnXnRqmvEtbbCyqdh767Qyh9xAoy+MHL3//wjWPd3CCWVS2oGnHwj5BQGLxsKVXh/NnjKI3M9r/QsOOn6iNWzvuEAK5//NamezyNyPWPiSXoPYtLl34/qPeK5KG8Qh25XWeYea+/4YURkJk6rhCFDhgQqEh/vPAKL7nOfSIdFQSG7MLLB4p3fwJq/hHZvgKzeMGlmZO5duQle/oH7JNj9w6GwZQlc/QKkpHbpSps+38f7f/oeXz3wLK0ayToaEx+f7DkaZ5PE6IlnsAj0v1Q7OH74QdVZwCyACRMmJEZGxO3vwn8egGMvhelzQIJ8GS35GSx5CFqaIDU9MnXwfA4Dx8PMJR2XU4UHj4CakBZwhqbavdYNr8KQUyJ33ZV/hpdug7d+BVN+ELx8O55bUcbCF/7C7NS/sWfk5Qy4+o+Rq6MxcXJ0DO4Rz9lQZRy6F3IJsKuD44mvrgoW3AgFQ+ArvwkeKAByig6+NlI8FZBTHLycCBQMhprtkbu3N/AURLilN/7rcPwVsOR/Yds7Yb+8rrGZO/+2hp/97Q1+mf44rUVHMeDKRyNbR2N6sHgGi5eAa9xZUacAte4ewa8CX3L3CO4DfMk9lthU4cVbYf9nTosiq1dor8t1v9TrKiJXl7rKg9cNpmBIhIPFdkhJh7wBkbsmOIHtwl9Bn+Hw3I1OQAzRx5/tY9pj7/D8yu081/9JeqccIO3KpyEjJ7J1NKYHi+bU2XnAMuBoESkTkRtF5GYRudktshBnj+RNOPsifwvAHdj+H2C5+3O/d7A7ob33e9i4EL70PzDoxODlvbwtgDC+/ILyVBxssQTTezDU7gheLlS1O6B3CaRE4Z9WZj5c/pQTDF+4xZlI0AFV5dnSHVz02NtU1zXxn4krGVK7HDn/59BvdOTrZ0wPFs3ZUDOCnFfg1nbOzQHmRKNeUbFzJbz2Ezj6fJh0c/DyviLdsmj0QHN9eC2Luko4sB8y87p8++bKT2nIGURWSytpqVEIGEeMhS//Lyy8E959HE79dsBingPN/OSFD/n7qp2cOrKIx09voM+zj8Dxl8N42zXUmHD1mBTlcdNQCwuuh7z+MO3x0MYpfLW1LCojUx9vCyWUMQs4OLZQuyPk37YbmlrYVulha7mHLRUetpR72Fqxn60VHl5p2cTilnE89du3ue+iY5k0IsQWTjhOvgm2vgH/vheGTIaSCYec/mjPXm6du5ItFR6++8WjuO2UPqT+4QzoMwwu/HX4n5ExxoJFl6jCP+6Amh1w/cudWwOQUwhI5FoW3uuE07IA5z10ECwWfrCbee9vZ0u5h1219Ycs4ejfK5MRxXl85dhC+n9Qw+jRx7JvRzNXznqXi8cN5Ifnj6Zfr6xOvqEAROCix+APZ8Dfroeb34RsZ93m86vKuOu5D+iVnc7cmyZx6vBCmHeV8/dy07+drixjTNgsWHTFiqdg3fPwxXthyKTOXSMl1fmii9SYhbeFEm7Lop3ps6rKrDe38NDLHzGiOJeTh/VhePFghvfNZURxLsOLc8nNdP8ZVWyCD+CE48by78un8Lslm/jDG1v494bP+c4XR3HtqcNIj1TXVHYBTH8S5nwZXvo2evkz/O6NLfzi1Y1MHlHEozPG0zc/E5b+Fj55Fab+wln8aIzpFAsWnbXnQ3jlLhj5BTj1jq5dK7c4ci0L78rp3BC7f3L7Oau4A8yIam1VHvjXBua8s5ULxh7Br644gcy0DhbE1brX6D2Y7IxUvv+lo7nsxBLu+8c6HvjXBp4t3cF9Fx3H5JER6poqmeAE6td+zMI5/8MvNk1g2riB/GL6CWSkpUDZCqer6pgLYeI3InNPY5KUZZ3tjAP7nXGKrN5wyR+6PvMnt2/kWhZt3VBBt9R1pKQEnBF1oLmF2+evYs47W7n+tGH89qrxHQcKOBhwfNZYDCvOZc51JzP7mgnUNbYwY/a7fHveKvbUNoT6jjrUMOFmPsg5hS/ueJSfnNTEr68Y5wSK+hpYcB3kD4Rpj9k4hTFdZMGiMxb+ACo+gcv+CHkhfil3JKcogt1QFZCaCRlhzGzyW2uxt6GJ6+Ys559rd3P31GO458IxpKSE8GVbsx1S0iD/iEMOiwjnjunPv783hTu+MIpX1+3hC79cwqw3N9PU0vH0147U1jdxzZOlXFN1Hc1Zhdy4+z5SmvY7Y0kvfdvJzTV9Ttt4hjGm8yxYhGv1PCfv0pT/huFnRuaakeyG8i7IC+c36YLBzgA38PneBq78w7ss31bFr688gW9OGYmEeq2aHdBrIKQG7t3MSk/lu+cexb+/O4XJI4v434Uf8cVfvcGfl22jvrEl9PoCe2obuOL3y1i1vZr7Zkwh96tPQ/U2+Md3YPkfYcNL8IWfwuCTw7quMSYwG7MIR/nH8K/vw7AzYMp/Re66OcVOuo/Wli4nyQtrQZ5XwRDwfM6W3eVc/fQHVNc1Mue6kznzqDBbTTXboWBo0GJDinL447Un85+PPuM3izbxkxfX8cvXP+brk4ZyzalD6Zff8cypTz7bx7Vz3mdvQzNPXT+R044sBgbCWT+ExQ84GXePPBcm3xZe/Y0x7bJgEY7/3O8k+7t0dte/1H3lFgMK9dWhT3ltT11F+Ndwv+C/P+ufHEgt4a8zJ3N8Se/w712zHUaeHXLxc47pz9lH96P002pmv7mFx5dsYtabW7ho3EBuOmM4xww4PGVK6bYqbny6lIy0FP76zVM4dqBPPc/4HmxfBhUfwyW/j84qcmOSlAWLUNVVwcZXYOJM6HVE8PLh8LYEPJ34ovfnqWBzS3/u/v0yBhfmMKKvM711RN9chhXlkpV+eJArrcllAjAyo5pHZl7B0KLc8O/b3Aj7djuD5WEQEU4eVsjJwwrZWuHhyXe28rfSMhasKOOMUcV844wRnDGqGBHh1XV7uH3eKgYVZPP0DRMZXOiX2yklFb72N2g+YHmfjIkwCxah+vA5aG2CcR1mMemcCKb8aNxXzpIKpaqwke1VdTy3suyQ84MKstuCx/DiXOqbWpj7WhXvZMC9Z+aR15lAAbC3DNAuZZsdXpzL/dOO43vnHsXc97bz1NJtXDPnfY7un89pRxbz1NKtjC0pYM51J1OYmxH4IimpFiiMiQILFqFa/RfofzwMOD7y145QMsH5Sz/hqpY6ivsP5JVvnUFaagqeA81srfC0/Wwpd9JyPL9yJ/sONAMwZdSR6M508uq7kAneHSCPRGrygpwMbj37SG46Yzj/WLObP761hTnvbOWcY/rx2FfHk5Nh/2yNiTX7XxeK8o2wa6WTwC4aItCyeH5VGY/+YxlXZcIFk45vS+KXm5nGcYN6c9ygQ8cgVJVKTyOf7W3g6P75yGODDn7hd0bbGovwuqE6kpmWyvSTSrjsxEFs/GwfR/bNi05yQmNMUBYsQrH6LyCpTsbSaGgbs+hcMsGXP9jN959dw1UlAuWQlh98FpOIUJyXSXFepnOgq/ta1GwHSYFeAXfA7RIRCTjYbYyJHfs1LZjWFlj7Vxh1LuT1i849UtOd1eCdaFks/uhzbp+/ivFD+nDP2W79OjNI3tVgUbvDCRSR2hrWGJNQLFgEs2WJM8vnhCgMbPvqRMqPpZsruPn/VnD0gHzmXHcyWU3VB68Vrt5DYP8eZyZRZ9RsD3smlDGm+7BgEcyaeZBVAEdPje59cooPJgEMwYpPq7np6VKGFuXwzA2T6J2d7rOXRScS9bXta1HWcbn21GyP/L7bxpiEYcGiIw17YcM/4bjLIC0zuvfKLXZSdYTgw521XPfk+/TLz+T/bpx0cBppXYWz/3VWJxbUeQem20lV3qGWZicPUwQHt40xicWCRUfWv+BsUTruq9G/V4jJBD/+bB9X/+k9emWlM/cbpxy6qZA31UdnMqz6boIUrr07QVusZWFMD2bBoiOr50HRKBh0UvTv5W1ZtLafhXVbhYev/fE90lNTmHvTJAYVZB9awJtEsDPyBzozvjozyB0gNbkxpmexYNGeqq2wfamzYjsWeyHkFDu/nTfUBDz94c5avjr7XZpbWpl70ySGFQdYad2ZJIJeqWnObKbOBAvvXhg2wG1Mj2XBoj1r5gMCY6+Mzf3aFuYdOm6hqjy9dBuX/m4pCvz5xkmM6t/OPtKdSSLoq2DIYZsghaRmOyDQu6Tz9zbGJDRblBdIa6szC2r4mbH7AvRNJlg8CnA297nrubW8/OEezjmmHw9ffkL7OZHAWdQX6t7bgRQMga1vhP+6mh3OhkfRngRgjIkbCxaBbF/mzAo6+4exu6dfyo81O2q4bd5Kdtc08MPzj+Gm00d0vFtdcyMcqO1iy2Kws6akuRHSOghK/mo+tZlQxvRwUe2GEpHzRGSjiGwSkbsCnB8qIotEZK2ILBGREp9z/09EPnR/YtQX5Fozz9mWdPRXYndPt0WgngrmvL2V6b9fSkuL8tdvTmbmmSODb2ta14U1Fl4FQ0BbndlN4bA1Fsb0eFELFiKSCjwOTAXGADNEZIxfsYeBZ1R1LHA/8JD72guAE4FxwCTgByISm+RAjXWw7gUYMw0yOpmuuzPcFsGL76zl/n+uZ8pRfVl4xxmcNDTE/aO90267OmYB4Q1yt7Y4wcWChTE9WjRbFhOBTaq6RVUbgfnANL8yY4BF7uPFPufHAG+oarOqeoA1wHlRrOtBH/0LGvdFP72Hn9W769lPDtUVu/jxBaOZfc0ECnLC6Arytiw6k+rDyzubKZxB7n27obXZZkIZ08NFM1gMAny/dcrcY77WAJe5jy8B8kWkyD0+VURyRKQYOBs47NtIRGaKSKmIlJaXh54qo0Nr/uLkSRp6WmSuF4Sq8se3tjD9iaXUSC+mjcrgpjNGIOFO1/VmrO3KAHevQU7m2HBaFrbGwpikEM1gEejbTv2e3wlMEZFVwBRgJ9Csqq8BC4GlwDxgGdB82MVUZ6nqBFWd0LdvF36j9tq7y0kceMJVMdu/+bH/bOKBf23g7GP6MeCIEgrZ27kL1UWgGyotw5nVFFawiNymR8aYxBXNb8QyDm0NlACHbMWmqrtU9VJVHQ/8yD1W6/75oKqOU9VzcQLPJ1Gsq2PtX50B3hOuivqtvP71wW4mDitk1tUnkZbXt9N7WuCpcFZgZxV0rUIFQ8JL+eENLLbGwpgeLZrBYjkwSkSGi0gGcBXwkm8BESkWEW8d7gbmuMdT3e4oRGQsMBZ4LYp1BVUnvcfgU6BoZFRv5VW5/wAf7dnHmUcVO91OuUWd3y2vrgJyCrveIgp3X4uaTyGvP6RnBy9rjOm2ohYsVLUZuA14FdgAPKuq60TkfhG5yC12FrBRRD4G+gMPusfTgbdEZD0wC/i6e73o2bUSKjY66T1i5N0tVQBMHul2HeUUOy0E9e+tC4GnomvjFV69Bzuzm1pC/Ouu3WGD28YkgaguylPVhThjD77H7vF5vABYEOB1DTgzomJn9TxIzYRjL4nZLZdtqSA3I5WxJW5K8dxiaG2CA3vDTzPelSSCvgqGODmq9u0KbRyiZjscMa7r9zXGJDTLDQXO7nAfLoBjLujcXhCdtHRzJROHF5Ke6n4M3pZBmDvmtb2mKwvyvMJZa9Ha6myWZIPbxvR4FiwAPn4V6qtjs2+Fa09tA1vKPZw60qc10E4ywZB4yiPXsoDQgsX+z6Cl0VJ9GJMELFiAk94jrz+MODtmt1y2xWk9TB7p0xrwTSYYjpYmJ7V5RMYs3FlNocyIaltjMbTr9zXGJDQLFp4K+OQ1GHuFs6dDjCzdVEnv7HRGH+GTxcQvmWDI6qoOfX1XpGWGvtai1tZYGJMsLOtsajqcez8ceW7MbqmqLN1cySkjCkn1TRDY2TGLSCzI89V7cGh7cXvL2BoLY3o8a1lk9YbJt0Lfo2J2yx1V9eysqT90vAIgIwfSc8MPFt7ykeiGgtA3QarZ7twzlgkXjTFxYcEiDrzjFaeODDB7qTML8yLdsigY7Mxyam3puFzNDhvcNiZJWLCIg6WbKynOy+TIfnmHn/QuzAtHJJII+ioY4mSS3ben43K2j4UxScOCRYx5xytOHVkUOLNsbnEnWxbipPuIhFCmz6o6XVUWLIxJChYsYmxz+X7K9x0I3AUFbssizHUWngrI7gMpqV2vIDgp2qHjYOEph+aGg2WNMT2aBYsYW7rZCQST2wsW3jGLcPJD1VVEbrwCDo5D1HYQLGwfC2OSigWLGFu6qZJBBdkMKcwJXCCn2PmNvdET+kU9lZEbrwAng2xuv45bFhYsjEkqFixiqLVVWbalksntjVdA5xbmecqdFkkkFQwOMVjYbChjkoEFixhav3svtfVN7Y9XgM/CvDDGLeoilJ7cV7BNkGq2O+MkmfmRva8xJiFZsIihd7cEGa+A8FsWrS1Ouo9IjlnAwYV5ra2Bz9tMKGOSigWLGFq6uZIRxbkc0buDXeVyw0z5UV8NKORGYA9yX70HOxll938W+HzNdtv0yJgkYsEiRppaWnnPHa/oUFs3VHloF25L9RHpMQs3k2ygtB+q7oI8yzZrTLKwYBEjH+ysxdPYEjxYZORCWlbo3VCRTvXh5R24DjTIXVcFTXU2uG1MErFgESPL3PUVp4wIEixEwluYF+kkgl7eLqZA2We9x2zMwpikYcEiRpZuruCYAfkU52UGLxxOMsFotSwy85yurUAzomyNhTFJx4JFDDQ0tVC6rTp4F5RXOMkE25IIRnjMAtx9LQJ0Q3nHMWyA25ikYcEiBlZtr+FAc+vh+1e0J5xkgnUVzp4cqemdr2B7CoYEDhY12yGzN2QXRP6expiEZMEiBpZtqSRFYOLwELPChjVmUR758Qov71oL/zxVNbbGwphkE9VgISLnichGEdkkIncFOD9URBaJyFoRWSIiJT7nfi4i60Rkg4g8Ku3mx0h8yzZXcPyg3vTODvG3/9wiaPJAU33wsp4IJxH0VTDEyVPlP423ZrvNhDImyUQtWIhIKvA4MBUYA8wQkTF+xR4GnlHVscD9wEPua08FTgPGAscBJwNTolXXaKprbGbV9homh9oFBeHtxV0X4SSCvgLta9G2xsJaFsYkk2i2LCYCm1R1i6o2AvOBaX5lxgCL3MeLfc4rkAVkAJlAOtDOUuLEtnxbNc2tGvrgNoSX8sNTEfkkgl6BgkVDDTTus2BhTJKJZrAYBPjOuyxzj/laA1zmPr4EyBeRIlVdhhM8drs/r6rqhijWNSSrd9Qw+80ttLSGvtfE0s0VpKUIJw/rE/qNvKk7go1btLY6LYtIp/rw6h1gYZ73sc2EMiapRDNYBBpj8P+WvROYIiKrcLqZdgLNInIkMBoowQkw54jImYfdQGSmiJSKSGl5eYjpMbrg/979lAcXbuD2eatoaGoJ6TXLNlcyfkgBORlpod/IOw02WMuioQa0JXrdUFm9IKvg0JQftsbCmKQUzWBRBvj++lkC7PItoKq7VPVSVR0P/Mg9VovTynhXVfer6n7gZeAU/xuo6ixVnaCqE/r2jdJv1z6qPI3kZKTyrw92c+2c96mtb+qwfG19Ex/urA1vvAJ8kgkGCYB1lYeWjwb/fS28i/QsWBiTVKIZLJYDo0RkuIhkAFcBL/kWEJFiEfHW4W5gjvt4O06LI01E0nFaHXHvhqryNHLS0D785qpxrNxezZV/WMae2oZ2y7+/tYpWpeP9KwLJ7AUp6cEHuKOVRNBXwdDDu6Ey8py9LIwxSSNqwUJVm4HbgFdxvuifVdV1InK/iFzkFjsL2CgiHwP9gQfd4wuAzcAHOOMaa1T1H9Gqa6iqPI0U5mYwbdwgnrxuIjuq6rjsiaVs+nxfwPJLN1eQmZbC+CFhLl4TCW1hXrRSffjyboLkXWvhnQnVfWcyG2M6IYyO9PCp6kJgod+xe3weL8AJDP6vawG+Gc26dUa1p5E+ORkAnD6qmL9+czLXPbmcy55YxpzrJnDS0EMX3S3bXMnJwwrJTEsN/2ahLMyLVhJBX70HO2s+6qqcWVe1to+FMcnIVnCHqLG5lX0HminKzWg7dtyg3vz9llMpzM3gq7Pf4/X1B2f3Vuw/wEd79oU3ZdZXKMkEPTFqWcDBTLO2xsKYpGTBIkTVdY0A9PEJFgBDinJYcPNkjhmQzzf/XMpf3nP690PaQrUjoSQTrKuAjHxICyGTbWd5A0PtDmiodX4sWBiTdKLaDdWTVHmcYFHoFywAivIymTfzFG6du5IfPv8Bn+1t4PN9B8jLTGPsoN6du2Fu8cHZTu2J5oI8L99NkNpmQlk3lDHJxoJFiKo7CBYAORlpzLpmAnf//QN+s+gTUlOEKUf1JS21k423nGI4sBeaD7TfcqiriO54BTjrLDJ7ucHC1lgYk6ysGypElUGCBUB6agq/mD6W284+kpZWZcpRXVj74W0xdNS68FRGd7wCnFlP3hlR3sV5tve2MUnHWhYh8o5ZdBQsAESEO798NBeNG8jIvnmdv2Fbyo8K6DUwcJm6Chh4QufvESrvJkg12yEtO7rrOowxCclaFiGq3O8Ei4IQ04wf1T+f1JQurEXICZJMUNUJJNHuhoKDmyDVfGprLIxJUhYsQlRd10jv7PTOj0GEKzdImvIDe6G1KfrdUOAEiMZ9sOcDG68wJklZsAhRlafxkDUWUeft6mkvWMRiQZ6Xd/ZT9TabCWVMkrJgEaIqT+NhayyiKqsAJLX9bqhYJBH08m1NWMvCmKQUNFi4O94lPW9eqJhJSXFaF+22LNyMtLEYbO7tEyAs1YcxSSmUlsUmEflFgC1Rk0qVp5HCnBgGC+h4YV4sUn145RRCeq7z2KbNGpOUQgkWY4GPgT+KyLvuhkO9olyvhKKqVNfFuBsKOm5Z1MVwzMK71gKsG8qYJBU0WKjqPlWdraqnAv8F/BTYLSJPuzva9Xj7DzTT1KKxHeCGjtOUeyohPQcycmJTl4LBkJoZvS1cjTEJLeiiPHfM4gLgemAY8EtgLnAGTvrxo6JYv4RIZMBkAAAaM0lEQVTgzQsV+5ZFB8kEY5Hqw9fR5zuBIsXmRBiTjEJZwf0JsBj4haou9Tm+INC+2D2RN1jEpWXRUAMtTZDqtxgwFkkEfU243vkxxiSlUILFWHcf7MOo6u0Rrk9Cil/Lwpsfqgry+x96rq4CcvvFtj7GmKQVSrBoFpFbgWOBLO9BVb0harVKMG3pyWM+G8odH6irODxYeCqh37GxrY8xJmmF0gH9Z2AA8GXgDaAECLzpdA/VlkQwLw7dUHBwTYWXqtuysIR+xpjYCCVYHKmqPwE8qvo0zmD38dGtVmKp9DSSkZpCbkaM1yfmtJMfqtEDzQ2xHeA2xiS1UIJFk/tnjYgcB/TGmRWVNKo9jfTJTUdinW3V27LwX5hXF8MFecYYQ2hjFrNEpA/wY+AlIA/4SVRrlWCqPE0U5kZxn+v2ZPcB5PCWRSyTCBpjDEGChYikAHtVtRp4ExgRk1olmCrPAQpzQ9vHIqJSUp1UG/4L82KZ6sMYYwjSDaWqrcBtMapLwqqui1PLAgIvzGtL9WED3MaY2AhlzOJ1EblTRAaLSKH3J5SLi8h5IrJRRDaJyF0Bzg8VkUUislZElohIiXv8bBFZ7fPTICIXh/neIqZy/wEKc+LQsoDAyQStZWGMibFQxiy86ylu9TmmBOmSctOEPA6cC5QBy0XkJVVd71PsYeAZVX1aRM4BHgKuVtXFwDj3OoXAJuC1EOoacU0trextaI79gjyvnCIo33josboKJ09TRhf2+DbGmDAEDRaqOryT154IbFLVLQAiMh+YBvgGizHAd93Hi4EXAlxnOvCyqtZ1sh5dUlPnTAaLeaoPr9xi+PSdQ495Kp3jthe2MSZGQkkkeE2g46r6TJCXDgJ2+DwvAyb5lVkDXAb8BrgEyBeRIlX17Xe5CvhVO3WbCcwEGDIkOqmz45bqwyun2En30driDHiDm0TQxiuMMbETypjFyT4/ZwD3AheF8LpAv/aq3/M7gSkisgqYAuwEmtsuIHIEzgLAVwPdQFVnqeoEVZ3Qt290Ume3pfqIW8uiL6BQX33wmKfCUoUbY2IqlG6ob/s+F5HeOClAgikDfPfgLAF2+V17F3Cpe9084DJVrfUpcgXwvKo2ESfxDxZuC8JT7rNIrwKKR8WnPsaYpNSZzQnqgFC+qZYDo0RkuIhk4HQnveRbQESK3bUcAHcDc/yuMQOY14k6RkxVXZySCHoFSvnhqbQFecaYmAplzOIfHOw+SsEZlH422OtUtVlEbsPpQkoF5qjqOhG5HyhV1ZeAs4CHRERxFv21zbgSkWE4LZM3wng/EVcd7zEL39YEQGMdNHksiaAxJqZCmTr7sM/jZuBTVS0L5eKquhBnNz3fY/f4PF4ALGjntdtwBsnjqsrTSH5WGumpcdohzr9lEcu9t40xxhVKsNgO7FbVBgARyRaRYe6XeY9X5WmM33gFOOk+4ODCPFuQZ4yJg1B+Xf4b0OrzvMU9lhSq6+IcLFLTIavAp2XhBg1rWRhjYiiUYJGmqo3eJ+7jOH57xlbl/sb4DW575RYf7H6yloUxJg5CCRblItK2rkJEpgEVHZTvUeLesoBDkwlaEkFjTByEMmZxMzBXRB5zn5cBAVd19zSqSmW8xyzAaUVUbXEeeyogJR2yese3TsaYpBLKorzNwCnuojlR1aTZf7uusYXG5tb4TZv1yimCHe+7lXJTfVheKGNMDAXthhKR/xWRAlXdr6r7RKSPiDwQi8rFW9xXb3vl9nUGtltb3SSClurDGBNboYxZTFXVGu8Td9e886NXpcTRFiwSYYBbW6ChxmlZ2II8Y0yMhRIsUkWkbZs4EckG4rRtXGy1pfrIi3c3lM/CPE+5TZs1xsRcKAPc/wcsEpEn3efXA09Hr0qJo2p/orQs3JZEXcXBvSyMMSaGQhng/rmIrAW+iJN2/BVgaLQrlgiq6+KcF8rL25LYuwsa91nLwhgTc6EmPNqDs4r7MuALwIao1SiBVHkaSUsRemWF0gCLIm9Lwru9qo1ZGGNirN1vQRE5Ciet+AygEvgrztTZs2NUt7ir8jTSJzcDifc0Ve8CvPKP3OfWsjDGxFZHvzJ/BLwFfEVVNwGIyHc7KN/jVHkSINUHQFomZPbyaVlYsDDGxFZH3VCX4XQ/LRaR2SLyBQJvldpjxT3jrK+cIqja7D62YGGMia12g4WqPq+qVwLHAEuA7wL9ReQJEflSjOoXV1WJkBfKK7cYWpsPPjbGmBgKOsCtqh5VnauqF+Lso70auCvqNUsA1QnVsnADhKQ6KcuNMSaGwtr+TVWrVPUPqnpOtCqUKFpalZr6pvhPm/XytiZyiiAlTrv2GWOSln3rtKOmrhFVKMxJj3dVHN5gYV1Qxpg4sGDRjuq2VB8Jktkkx6dlYYwxMWbBoh2ViZLqw8taFsaYOLJg0Y62lkWijFm0tSwsWBhjYs+CRTsqE2UvCy9vig9rWRhj4sCCRTuqPd4kggkywJ1/BCDQa2C8a2KMSUJRDRYicp6IbBSRTSJy2NoMERkqIotEZK2ILBGREp9zQ0TkNRHZICLrRWRYNOvqr8rTRF5mGplpqbG8bfvyB8ANr8DxV8S7JsaYJBS1YCEiqcDjwFRgDDBDRMb4FXsYeEZVxwL3Aw/5nHsG+IWqjgYmAp9Hq66BVHkOJE6rwmvIKZCeFe9aGGOSUDRbFhOBTaq6RVUbgfnANL8yY4BF7uPF3vNuUElT1dcB3P2/66JY18NU1TVRmJsg02aNMSbOohksBgE7fJ6Xucd8rcFJWAhwCZAvIkXAUUCNiPxdRFaJyC/clsohRGSmiJSKSGl5eXlEK1/lOZA4C/KMMSbOohksAmWoVb/ndwJTRGQVMAXYCTTjpE4/wz1/MjACuO6wi6nOUtUJqjqhb9++Eaw6VHsSKNWHMcbEWTSDRRkw2Od5CbDLt4Cq7lLVS1V1PPAj91it+9pVbhdWM/ACcGIU63qYKk8jRRYsjDEGiG6wWA6MEpHhIpKBs+veS74FRKRYRLx1uBuY4/PaPiLibS6cA6yPYl0PUd/YQn1Ti7UsjDHGFbVg4bYIbgNexdmz+1lVXSci94vIRW6xs4CNIvIx0B940H1tC04X1CIR+QCnS2t2tOrqr6ouwVJ9GGNMnHW0rWqXqepCYKHfsXt8Hi8AFrTz2teBsdGsX3uq9ifY6m1jjIkzW8EdQFWi5YUyxpg4s2ARQHWi5YUyxpg4s2ARQMIlETTGmDizYBFAtaeR1BShV5YtyjPGGLBgEVBVXSN9ctJJSQm0rtAYY5KPBYsAqvY30semzRpjTBsLFgFU1TXaeIUxxviwYBFAlceChTHG+LJgEUC1p9FSfRhjjA8LFn5aW5XqOksiaIwxvixY+Kmtb6JVsQFuY4zxYcHCj6X6MMaYw1mw8FNlq7eNMeYwFiz8WLAwxpjDWbDwY0kEjTHmcBYs/HiTCNoAtzHGHGTBwk+1p5Hs9FSyM1LjXRVjjEkYFiz8WKoPY4w5nAULP5bqwxhjDmfBwk+1BQtjjDmMBQs/lRYsjDHmMBYs/FR7bC8LY4zxZ8HCR0NTC57GForyLFgYY4wvCxY+qutsjYUxxgQS1WAhIueJyEYR2SQidwU4P1REFonIWhFZIiIlPudaRGS1+/NSNOvpdTDVR3osbmeMMd1GWrQuLCKpwOPAuUAZsFxEXlLV9T7FHgaeUdWnReQc4CHgavdcvaqOi1b9AjkYLDJjeVtjjEl40WxZTAQ2qeoWVW0E5gPT/MqMARa5jxcHOB9T1rIwxpjAohksBgE7fJ6Xucd8rQEucx9fAuSLSJH7PEtESkXkXRG5ONANRGSmW6a0vLy8yxWutpaFMcYEFM1gIQGOqd/zO4EpIrIKmALsBJrdc0NUdQLwVeARERl52MVUZ6nqBFWd0Ldv3y5XuMrTiAj0zraWhTHG+IramAVOS2Kwz/MSYJdvAVXdBVwKICJ5wGWqWutzDlXdIiJLgPHA5ijWl6q6Rgqy00lNCRTnjDEmeUWzZbEcGCUiw0UkA7gKOGRWk4gUi4i3DncDc9zjfUQk01sGOA3wHRiPCssLZYwxgUUtWKhqM3Ab8CqwAXhWVdeJyP0icpFb7Cxgo4h8DPQHHnSPjwZKRWQNzsD3z/xmUUWFBQtjjAksmt1QqOpCYKHfsXt8Hi8AFgR43VLg+GjWLZBqTxPDinNifVtjjEl4UQ0W3U2lp5EThxbEuxrGJL2mpibKyspoaGiId1V6jKysLEpKSkhP79wEHgsWLlWlus6SCBqTCMrKysjPz2fYsGGI2ISTrlJVKisrKSsrY/jw4Z26huWGcu1taKalVW3MwpgE0NDQQFFRkQWKCBERioqKutRSs2DhOrh624KFMYnAAkVkdfXv04KFyxss+liwMMaYw1iwcHmDRZEFC2OSXmVlJePGjWPcuHEMGDCAQYMGtT1vbGwM6RrXX389GzdujHJNY8cGuF3evFA2wG2MKSoqYvXq1QDce++95OXlceeddx5SRlVRVVJSAv/O/eSTT0a9nrFkwcJV5W58ZLvkGZNY7vvHOtbv2hvRa44Z2IuffuXYsF+3adMmLr74Yk4//XTee+89/vnPf3LfffexcuVK6uvrufLKK7nnHmcp2emnn85jjz3GcccdR3FxMTfffDMvv/wyOTk5vPjii/Tr1y+i7ynarBvKVeVpJDMthez01HhXxRiTwNavX8+NN97IqlWrGDRoED/72c8oLS1lzZo1vP7666xff3iyidraWqZMmcKaNWuYPHkyc+bMiUPNu8ZaFi5vqg+bgWFMYulMCyCaRo4cycknn9z2fN68efzpT3+iubmZXbt2sX79esaMGXPIa7Kzs5k6dSoAJ510Em+99VZM6xwJFixclhfKGBOK3NzctseffPIJv/nNb3j//fcpKCjg61//esC1DBkZB79bUlNTaW5uPqxMorNuKJcFC2NMuPbu3Ut+fj69evVi9+7dvPrqq/GuUtRYy8JVXdfI0CJLImiMCd2JJ57ImDFjOO644xgxYgSnnXZavKsUNaLqv3ld9zRhwgQtLS3t9OuP/+mrXHZSCfdelFj9o8Ykow0bNjB69Oh4V6PHCfT3KiIr3F1JO2TdUEBjcyv7DjRbN5QxxrTDggVOFxRYXihjjGmPBQssiaAxxgRjwYKDqT4sWBhjTGAWLHB2yAMLFsYY0x4LFhwcs7AkgsYYE5gFC3z2ssjp3N60xpie5ayzzjpsgd0jjzzCt771rXZfk5eXB8CuXbuYPn16u9cNNsX/kUceoa6uru35+eefT01NTahVjxoLFjjBond2Ommp9tdhjIEZM2Ywf/78Q47Nnz+fGTNmBH3twIEDWbBgQafv7R8sFi5cSEFBQaevFym2ghtL9WFMQnv5LtjzQWSvOeB4mPqzdk9Pnz6dH//4xxw4cIDMzEy2bdvGrl27GDduHF/4wheorq6mqamJBx54gGnTph3y2m3btnHhhRfy4YcfUl9fz/XXX8/69esZPXo09fX1beVuueUWli9fTn19PdOnT+e+++7j0UcfZdeuXZx99tkUFxezePFihg0bRmlpKcXFxfzqV79qy1h700038Z3vfIdt27YxdepUTj/9dJYuXcqgQYN48cUXyc7OjuhfWVR/lRaR80Rko4hsEpG7ApwfKiKLRGStiCwRkRK/871EZKeIPBbNelqwMMb4KioqYuLEibzyyiuA06q48soryc7O5vnnn2flypUsXryY73//+3SUBeOJJ54gJyeHtWvX8qMf/YgVK1a0nXvwwQcpLS1l7dq1vPHGG6xdu5bbb7+dgQMHsnjxYhYvXnzItVasWMGTTz7Je++9x7vvvsvs2bNZtWoV4CQ0vPXWW1m3bh0FBQU899xzEf87iVrLQkRSgceBc4EyYLmIvKSqvsneHwaeUdWnReQc4CHgap/z/wO8Ea06elV5GinpY3mhjElIHbQAosnbFTVt2jTmz5/PnDlzUFV++MMf8uabb5KSksLOnTv57LPPGDBgQMBrvPnmm9x+++0AjB07lrFjx7ade/bZZ5k1axbNzc3s3r2b9evXH3Le39tvv80ll1zSlvX20ksv5a233uKiiy5i+PDhjBs3DnBSoG/bti1CfwsHRbNlMRHYpKpbVLURmA9M8yszBljkPl7se15ETgL6A69FsY6AMxvK9t42xvi6+OKLWbRoUdsueCeeeCJz586lvLycFStWsHr1avr37x8wJbmvQHvkbN26lYcffphFixaxdu1aLrjggqDX6agFk5mZ2fY4WinQoxksBgE7fJ6Xucd8rQEucx9fAuSLSJGIpAC/BH7Q0Q1EZKaIlIpIaXl5eacqqapUeRrpY8HCGOMjLy+Ps846ixtuuKFtYLu2tpZ+/fqRnp7O4sWL+fTTTzu8xplnnsncuXMB+PDDD1m7di3gpDbPzc2ld+/efPbZZ7z88sttr8nPz2ffvn0Br/XCCy9QV1eHx+Ph+eef54wzzojU2w0qmsEi0JZz/qHxTmCKiKwCpgA7gWbgW8BCVd1BB1R1lqpOUNUJffv27VQl9x9opqlFKcy1abPGmEPNmDGDNWvWcNVVVwHwta99jdLSUiZMmMDcuXM55phjOnz9Lbfcwv79+xk7diw///nPmThxIgAnnHAC48eP59hjj+WGG244JLX5zJkzmTp1KmefffYh1zrxxBO57rrrmDhxIpMmTeKmm25i/PjxEX7H7YtainIRmQzcq6pfdp/fDaCqD7VTPg/4SFVLRGQucAbQCuQBGcDvVPWwQXKvzqYor/Y0cs9L67j8pBLOPKpzAccYE1mWojw6upKiPJpTZ5cDo0RkOE6L4Srgq74FRKQYqFLVVuBuYA6Aqn7Np8x1wISOAkVX9MnN4LczYhedjTGmO4paN5SqNgO3Aa8CG4BnVXWdiNwvIhe5xc4CNorIxziD2Q9Gqz7GGGM6L6qL8lR1IbDQ79g9Po8XAB0udVTVp4CnolA9Y0wCU9WAM4lM53R1yMHyWxhjEk5WVhaVlZVd/oIzDlWlsrKSrKysTl/D0n0YYxJOSUkJZWVldHZKvDlcVlYWJSUlwQu2w4KFMSbhpKenM3z48HhXw/iwbihjjDFBWbAwxhgTlAULY4wxQUVtBXesiUg50HGilo4VAxURqk4i6GnvB3ree+pp7wd63nvqae8HDn9PQ1U1aPqKHhMsukpESkNZ8t5d9LT3Az3vPfW09wM97z31tPcDnX9P1g1ljDEmKAsWxhhjgrJgcdCseFcgwnra+4Ge95562vuBnveeetr7gU6+JxuzMMYYE5S1LIwxxgRlwcIYY0xQSR8sROQ8EdkoIptEJCobLMWaiGwTkQ9EZLWIhL99YJyJyBwR+VxEPvQ5Vigir4vIJ+6ffeJZx3C1857uFZGd7ue0WkTOj2cdwyEig0VksYhsEJF1InKHe7xbfk4dvJ/u/Bllicj7IrLGfU/3uceHi8h77mf0VxHJCOl6yTxmISKpwMfAuUAZzu5+M1R1fVwr1kUisg1nd8FuuZhIRM4E9gPPqOpx7rGf4+yq+DM3qPdR1f+OZz3D0c57uhfYr6oPx7NunSEiRwBHqOpKEckHVgAXA9fRDT+nDt7PFXTfz0iAXFXdLyLpwNvAHcD3gL+r6nwR+T2wRlWfCHa9ZG9ZTAQ2qeoWVW0E5gPT4lynpKeqbwJVfoenAU+7j5/G+Y/cbbTznrotVd2tqivdx/twdsMcRDf9nDp4P92WOva7T9PdHwXO4eCmcyF/RskeLAYBO3yel9HN/4G4FHhNRFaIyMx4VyZC+qvqbnD+YwP94lyfSLlNRNa63VTdosvGn4gMA8YD79EDPie/9wPd+DMSkVQRWQ18DrwObAZq3G2vIYzvvGQPFoH2bOwJ/XKnqeqJwFTgVrcLxCSeJ4CRwDhgN/DL+FYnfCKSBzwHfEdV98a7Pl0V4P10689IVVtUdRxQgtOTMjpQsVCulezBogwY7PO8BNgVp7pEjKrucv/8HHge5x9Jd/eZ26/s7V/+PM716TJV/cz9z9wKzKabfU5uP/hzwFxV/bt7uNt+ToHeT3f/jLxUtQZYApwCFIiId+O7kL/zkj1YLAdGubMDMoCrgJfiXKcuEZFcd4AOEckFvgR82PGruoWXgGvdx9cCL8axLhHh/VJ1XUI3+pzcwdM/ARtU9Vc+p7rl59Te++nmn1FfESlwH2cDX8QZi1kMTHeLhfwZJfVsKAB3KtwjQCowR1UfjHOVukRERuC0JsDZNvcv3e09icg84CycVMqfAT8FXgCeBYYA24HLVbXbDBi3857OwuneUGAb8E1vf3+iE5HTgbeAD4BW9/APcfr5u93n1MH7mUH3/YzG4gxgp+I0DJ5V1fvd74j5QCGwCvi6qh4Ier1kDxbGGGOCS/ZuKGOMMSGwYGGMMSYoCxbGGGOCsmBhjDEmKAsWxhhjgrJgYUwYRKTFJwPp6khmKhaRYb5ZaY1JJGnBixhjfNS76ROMSSrWsjAmAtw9RP6fu3/A+yJypHt8qIgschPRLRKRIe7x/iLyvLvXwBoROdW9VKqIzHb3H3jNXXlrTNxZsDAmPNl+3VBX+pzbq6oTgcdwsgLgPn5GVccCc4FH3eOPAm+o6gnAicA69/go4HFVPRaoAS6L8vsxJiS2gtuYMIjIflXNC3B8G3COqm5xE9LtUdUiEanA2VSnyT2+W1WLRaQcKPFNs+Cmxn5dVUe5z/8bSFfVB6L/zozpmLUsjIkcbedxe2UC8c3R04KNK5oEYcHCmMi50ufPZe7jpTjZjAG+hrO1JcAi4BZo26CmV6wqaUxn2G8txoQn2915zOsVVfVOn80Ukfdwfgmb4R67HZgjIj8AyoHr3eN3ALNE5EacFsQtOJvrGJOQbMzCmAhwxywmqGpFvOtiTDRYN5QxxpigrGVhjDEmKGtZGGOMCcqChTHGmKAsWBhjjAnKgoUxxpigLFgYY4wJ6v8Dx2syvYqZvAcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_history(history[best_index][\"fit_history\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      1740\n",
      "           1       1.00      1.00      1.00       344\n",
      "\n",
      "   micro avg       1.00      1.00      1.00      2084\n",
      "   macro avg       1.00      1.00      1.00      2084\n",
      "weighted avg       1.00      1.00      1.00      2084\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model(history[best_index][\"model\"], train_sequences, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00       192\n",
      "           1       1.00      1.00      1.00        39\n",
      "\n",
      "   micro avg       1.00      1.00      1.00       231\n",
      "   macro avg       1.00      1.00      1.00       231\n",
      "weighted avg       1.00      1.00      1.00       231\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model(history[best_index][\"model\"], valid_sequences, valid_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00       480\n",
      "           1       0.98      0.98      0.98        98\n",
      "\n",
      "   micro avg       0.99      0.99      0.99       578\n",
      "   macro avg       0.99      0.99      0.99       578\n",
      "weighted avg       0.99      0.99      0.99       578\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model(history[best_index][\"model\"], test_sequences, test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "for entry in history:\n",
    "    clean_model(entry[\"model\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>BiLSTM on Pretrained (Glove + ELMo) Embeddings (Not Working)</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ELMo(Layer):\n",
    "    \n",
    "    def __init__(self, index_word, output_mode=\"default\", **kwargs):\n",
    "        \n",
    "        assert output_mode in [\"default\", \"word_emb\", \"lstm_outputs1\", \"lstm_outputs2\", \"elmo\"]\n",
    "        \n",
    "        self.index_word = index_word\n",
    "        self.output_mode = output_mode\n",
    "        \n",
    "        super().__init__(**kwargs)\n",
    "    \n",
    "    \n",
    "    def build(self, input_shape):\n",
    "        \n",
    "        self.max_length = input_shape[1]\n",
    "        \n",
    "        self.word_mapping = [x[1] for x in sorted(self.index_word.items(), key=operator.itemgetter(0))]\n",
    "        self.lookup_table = tf.contrib.lookup.index_to_string_table_from_tensor(self.word_mapping, default_value=\"<UNK>\")\n",
    "        tf.tables_initializer().run(session=K.get_session())\n",
    "        \n",
    "        self.elmo_model = hub.Module(\"https://tfhub.dev/google/elmo/2\", trainable=self.trainable)\n",
    "        \n",
    "        super().build(input_shape)\n",
    "        \n",
    "        \n",
    "    def call(self, x):\n",
    "        x = tf.cast(x, dtype=tf.int64)\n",
    "        \n",
    "        sequence_lengths = tf.cast(tf.count_nonzero(x, axis=1), dtype=tf.int32)\n",
    "        strings = self.lookup_table.lookup(x)\n",
    "        \n",
    "        inputs = {\n",
    "            \"tokens\": strings,\n",
    "            \"sequence_len\": sequence_lengths\n",
    "        }\n",
    "\n",
    "        y = self.elmo_model(\n",
    "            inputs,\n",
    "            signature=\"tokens\",\n",
    "            as_dict=True\n",
    "        )[self.output_mode]\n",
    "        \n",
    "        with tf.Session() as sess:\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "            sess.run(tf.tables_initializer())\n",
    "            # return average of ELMo features\n",
    "            return sess.run(tf.reduce_mean(y, 1))\n",
    "    \n",
    "    \n",
    "    def compute_output_shape(self, input_shape):\n",
    "        \n",
    "        if self.output_mode == \"default\":\n",
    "            return (input_shape[0], 1024)\n",
    "        if self.output_mode == \"word_emb\":\n",
    "            return (input_shape[0], self.max_length, 512)\n",
    "        if self.output_mode == \"lstm_outputs1\":\n",
    "            return (input_shape[0], self.max_length, 1024)\n",
    "        if self.output_mode == \"lstm_outputs2\":\n",
    "            return (input_shape[0], self.max_length, 1024)\n",
    "        if self.output_mode == \"elmo\":\n",
    "            return (input_shape[0], self.max_length, 1024)\n",
    "        \n",
    "        \n",
    "    def get_config(self):\n",
    "        config = {\n",
    "            \"word_index\": self.word_index,\n",
    "            \"output_mode\": self.output_mode \n",
    "        }\n",
    "        \n",
    "        return list(config.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential(name=\"BiLSTM\")\n",
    "\n",
    "model.add(InputLayer((1,), dtype=tf.string))\n",
    "model.add(Lambda(elmo_vectors))\n",
    "#model.add(ELMo(tokenizer.index_word, trainable=False))\n",
    "#model.add(Bidirectional(CuDNNLSTM(100)))\n",
    "model.add(Dense(1, activation=\"sigmoid\"))\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer=\"adam\", loss=\"binary_crossentropy\", metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.tables_initializer().run(session=K.get_session())\n",
    "history = model.fit(\n",
    "    train_sequences, train_y,\n",
    "    validation_data=(valid_sequences, valid_y),\n",
    "    batch_size=1,\n",
    "    epochs=1,\n",
    "    #callbacks=[EarlyStopping(patience=6, restore_best_weights=True, verbose=1)],\n",
    "    verbose=1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_history(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluate_model(model, train_sequences, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluate_model(model, valid_sequences, valid_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluate_model(model, test_sequences, test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_model(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}