NLP_PS3_temp.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import operator\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import csv\n",
    "\n",
    "from itertools import chain, islice\n",
    "from functools import partial\n",
    "\n",
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "import keras\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Embedding, Flatten, Dense, Dropout\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras import backend as K\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from pprint import pprint\n",
    "\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_seed(seed):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    tf.set_random_seed(seed)\n",
    "    \n",
    "set_seed(1992)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Preprocessing</h2>\n",
    "<p>We load the <i>LingSpam</i> dataset and split the data in training, validation and test sets</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "root = \"lingspam_public/bare\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ham: 2412, Spam: 481\n"
     ]
    }
   ],
   "source": [
    "ham, spam = [], []\n",
    "\n",
    "for dirpath, _, filenames in os.walk(root):\n",
    "    for filename in filenames:\n",
    "        with open(os.path.join(dirpath, filename), encoding=\"utf8\") as file:\n",
    "            message = file.read()\n",
    "        if filename.startswith(\"spm\"):\n",
    "            spam.append(message)\n",
    "        else:\n",
    "            ham.append(message)\n",
    "            \n",
    "print(\"Ham: {}, Spam: {}\".format(len(ham), len(spam)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split Sizes: [2084, 231, 578]\n"
     ]
    }
   ],
   "source": [
    "def data_split(x, y, sizes=(0.1, 0.1)):\n",
    "    \n",
    "    data_size = len(x)\n",
    "    splits = []\n",
    "    \n",
    "    for size in sizes:\n",
    "        x, test_x, y, test_y = train_test_split(x, y, test_size=int(data_size*size), shuffle=True)\n",
    "        splits.append((test_x, test_y))\n",
    "    splits.append((x, y))\n",
    "    \n",
    "    return splits\n",
    "\n",
    "x = ham + spam\n",
    "y = [0] * len(ham) + [1] * len(spam)\n",
    "\n",
    "(test_x, test_y), (valid_x, valid_y), (train_x, train_y) = data_split(x, y, sizes=(0.2, 0.08))\n",
    "\n",
    "print(\"Split Sizes: {}\".format(list(map(len, (train_x, valid_x, test_x)))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Tokenizer Initialization</h3>\n",
    "<p>We consider three tokenizers. <i>Tokenizer_GLB</i> is fitted in all token in both the training and validation data. <i>Tokenizer_TRAIN_GLB</i> is fitted in all tokens onservable in the training set. <i>Tokenizer_UNK</i> is fitted in a subset of the training sequences in order to simulate the unobservability of a number of tokens in the validation data.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokens discovered by Tokenizer_GLB: 53917\n"
     ]
    }
   ],
   "source": [
    "tokenizer_GLB = Tokenizer(num_words=None, lower=True, oov_token=\"*UNK*\")\n",
    "tokenizer_GLB.fit_on_texts(chain(train_x, valid_x))\n",
    "\n",
    "print(\"Tokens discovered by Tokenizer_GLB: {}\".format(len(tokenizer_GLB.word_index)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokens discovered by Tokenizer_TRAIN_GLB: 50155\n"
     ]
    }
   ],
   "source": [
    "tokenizer_TRAIN_GLB = Tokenizer(num_words=None, lower=True, oov_token=\"*UNK*\")\n",
    "tokenizer_TRAIN_GLB.fit_on_texts(train_x)\n",
    "\n",
    "print(\"Tokens discovered by Tokenizer_TRAIN_GLB: {}\".format(len(tokenizer_TRAIN_GLB.word_index)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokens discovered by Tokenizer_UNK: 46737\n"
     ]
    }
   ],
   "source": [
    "tokenizer_UNK = Tokenizer(num_words=None, lower=True, oov_token=\"*UNK*\")\n",
    "tokenizer_UNK.fit_on_texts(random.sample(train_x, len(train_x) - len(valid_x))) # We want the number of unknown words in the train set to be approximately the same as in the validation set.\n",
    "\n",
    "print(\"Tokens discovered by Tokenizer_UNK: {}\".format(len(tokenizer_UNK.word_index)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Tokenization</h3>\n",
    "<p>We transform each set in sequences of tokens according the chosen tokenizer.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = tokenizer_UNK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sequences = pad_sequences(tokenizer.texts_to_sequences(train_x), maxlen=150, padding=\"post\", truncating=\"post\") # We are contrained by the hardware limitations.\n",
    "input_length = len(train_sequences[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_sequences = pad_sequences(tokenizer.texts_to_sequences(valid_x), maxlen=input_length, padding=\"post\", truncating=\"post\")\n",
    "test_sequences  = pad_sequences(tokenizer.texts_to_sequences(test_x), maxlen=input_length, padding=\"post\", truncating=\"post\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Utilities</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_model(model):\n",
    "    \n",
    "    K.clear_session()\n",
    "    del model\n",
    "    gc.collect()\n",
    "    \n",
    "    \n",
    "def plot_history(history):\n",
    "    \n",
    "    plt.plot(\n",
    "        history.history[\"acc\"],\n",
    "        label=\"Train\"\n",
    "    )\n",
    "    plt.plot(\n",
    "        history.history[\"val_acc\"],\n",
    "        label=\"Validation\"\n",
    "    )\n",
    "    \n",
    "    plt.title(\"Model Accuracy\")\n",
    "    plt.ylabel(\"Accuray\")\n",
    "    plt.xlabel(\"Epoch\")\n",
    "    plt.legend([\"Train\", \"Validation\"], loc=\"best\")\n",
    "    \n",
    "    plt.show()\n",
    "    \n",
    "\n",
    "def evaluate_model(model, sequences, y):\n",
    "    \n",
    "    print(classification_report(y, (model.predict(sequences).flatten() >= 0.5).astype(float)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>MLP on Pretrained (Glove) Embeddings</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_embeddings_path = \"glove.6B/glove.6B.100d.txt\"\n",
    "embeddings_df = pd.read_csv(glove_embeddings_path, sep=\" \", index_col=0, header=None, quoting=csv.QUOTE_NONE, na_values=None, keep_default_na=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Non-Zero Embeddings: 31799\n"
     ]
    }
   ],
   "source": [
    "token_iter = map(\n",
    "    operator.itemgetter(0),\n",
    "    sorted(\n",
    "        filter(lambda item: item[1] <= tokenizer.num_words, tokenizer.word_index.items()) if tokenizer.num_words\n",
    "            else tokenizer.word_index.items(),\n",
    "        key=operator.itemgetter(1)\n",
    "    )\n",
    ")\n",
    "\n",
    "embedding_weights = embeddings_df.reindex(chain((None,), token_iter), fill_value=0.0).values # The zero-indexed vector is used only for masking by the layer.\n",
    "\n",
    "print(\"Non-Zero Embeddings: {}\".format(np.count_nonzero(embedding_weights, axis=0)[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 150, 100)          4673800   \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 15000)             0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 100)               1500100   \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 6,174,001\n",
      "Trainable params: 1,500,201\n",
      "Non-trainable params: 4,673,800\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = Sequential(name=\"MLP\")\n",
    "\n",
    "model.add(Embedding(\n",
    "    input_dim=embedding_weights.shape[0],\n",
    "    output_dim=embedding_weights.shape[1],\n",
    "    input_length=input_length,\n",
    "    embeddings_initializer=\"zeros\",\n",
    "    weights=[embedding_weights],\n",
    "    trainable=False\n",
    "))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(100, activation=\"relu\"))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(1, activation=\"sigmoid\"))\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer=\"adam\", loss=\"binary_crossentropy\", metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 2084 samples, validate on 231 samples\n",
      "Epoch 1/100\n",
      "2084/2084 [==============================] - 2s 822us/step - loss: 0.4411 - acc: 0.8637 - val_loss: 0.0988 - val_acc: 0.9827\n",
      "Epoch 2/100\n",
      "2084/2084 [==============================] - 1s 431us/step - loss: 0.1110 - acc: 0.9597 - val_loss: 0.0525 - val_acc: 0.9827\n",
      "Epoch 3/100\n",
      "2084/2084 [==============================] - 1s 429us/step - loss: 0.0624 - acc: 0.9760 - val_loss: 0.0285 - val_acc: 1.0000\n",
      "Epoch 4/100\n",
      "2084/2084 [==============================] - 1s 438us/step - loss: 0.0425 - acc: 0.9866 - val_loss: 0.0241 - val_acc: 0.9913\n",
      "Epoch 5/100\n",
      "2084/2084 [==============================] - 1s 421us/step - loss: 0.0386 - acc: 0.9880 - val_loss: 0.0319 - val_acc: 0.9870\n",
      "Epoch 6/100\n",
      "2084/2084 [==============================] - 1s 445us/step - loss: 0.0299 - acc: 0.9894 - val_loss: 0.0316 - val_acc: 0.9913\n",
      "Epoch 7/100\n",
      "2084/2084 [==============================] - 1s 437us/step - loss: 0.0201 - acc: 0.9928 - val_loss: 0.0332 - val_acc: 0.9870\n",
      "Restoring model weights from the end of the best epoch\n",
      "Epoch 00007: early stopping\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(\n",
    "    train_sequences, train_y,\n",
    "    validation_data=(valid_sequences, valid_y),\n",
    "    batch_size=16,\n",
    "    epochs=100,\n",
    "    callbacks=[EarlyStopping(patience=3, restore_best_weights=True, verbose=1)],\n",
    "    verbose=1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8leWZ8PHflT1kIUAWSEIIKCiIGDCiVgVcK24IOmNpbdXaOrW1tp06rZ361urotG/H6dhWR4uKSutbxlFwK2qVIrgLJICyChggCyQEErKv1/vH8wQOhyyHkCcn5+T6fj7nk2c/1wnkuc69PPctqooxxhjTnYhgB2CMMWbgs2RhjDGmR5YsjDHG9MiShTHGmB5ZsjDGGNMjSxbGGGN6ZMnCDGoikisiKiJRARx7s4i81x9xGTPQWLIwIUNEikSkWURS/bavc2/4ucGJ7KhYEkSkVkSWBTsWY/qSJQsTar4A5nesiMjpQHzwwjnG9UATcJmIjOrPNw6kdGRMb1myMKHmT8A3fNZvAhb5HiAiQ0VkkYhUiMguEblHRCLcfZEi8pCI7BeRncCVnZz7lIiUiUiJiDwgIpHHEd9NwOPABuBrftceLSJL3LgqReQRn33fFpHNIlIjIptEZJq7XUXkZJ/jnhGRB9zlWSJSLCI/FZG9wNMiMkxEXnPf46C7nO1z/nAReVpESt39L7nbPxORq32Oi3Z/R3nH8dlNGLNkYULNR0CyiEx0b+I3AH/2O+YPwFBgHDATJ7nc4u77NnAVMBXIxykJ+HoWaAVOdo+5DPhWIIGJSA4wC3jOfX3DZ18k8BqwC8gFsoDF7r5/AH7pHp8MXANUBvKewEhgODAGuA3nb/ppdz0HaAAe8Tn+T8AQ4DQgHfgvd/si4Eaf464AylR1XYBxmHCnqvayV0i8gCLgEuAe4FfA5cBbQBSgODfhSJxqoEk+5/0T8I67/HfgOz77LnPPjQIy3HPjffbPB1a4yzcD73UT3z3AOnc5E2gDprrr5wIVQFQn570J/KCLaypwss/6M8AD7vIsoBmI6yamPOCguzwKaAeGdXJcJlADJLvrLwA/Cfa/ub0GzsvqOE0o+hOwChiLXxUUkArE4HyD77AL55s8ODfFPX77OowBooEyEenYFuF3fHe+ATwBoKqlIrISp1qqEBgN7FLV1k7OGw3sCPA9/FWoamPHiogMwSktXA4MczcnuSWb0cABVT3ofxE33veB60RkKTAb+EEvYzJhyKqhTMhR1V04Dd1XAEv8du8HWnBu/B1ygBJ3uQznpum7r8MenJJFqqqmuK9kVT2tp5hE5EvAeOBnIrLXbUM4G5jvNjzvAXK6aITeA5zUxaXrcaqNOoz02+8/bPSPgVOAs1U1GZjREaL7PsNFJKWL93oWpyrqH4APVbWki+PMIGTJwoSqW4GLVLXOd6OqtgHPAw+KSJKIjAH+mSPtGs8Dd4pItogMA+72ObcM+BvwnyKSLCIRInKSiMwMIJ6bcKrEJuFU/eQBk3Fu9LOBT3AS1a/d7rVxInKee+6TwF0icqY4TnbjBlgHfNVtmL8cpw2mO0k47RRVIjIcuNfv870O/LfbEB4tIjN8zn0JmIZTovAvsZlBzpKFCUmqukNV13Sx+/tAHbATeA/4f8BCd98TOG0E64ECji2ZfAOnGmsTcBCn7r7bLrAiEgf8I/AHVd3r8/oCp8rsJjeJXY3TcL4bKMZpnEdV/xd40I2zBuemPdy9/A/c86pwele91F0swMM4XYn343QGeMNv/9dxSl5bgHLghx07VLUBeBGnes//92IGOVG1yY+MMQ4R+QUwQVVv7PFgM6hYA7cxBnCewcCp3vt6sGMxA49VQxljEJFv4zSAv66qq4Idjxl4rBrKGGNMj6xkYYwxpkdh02aRmpqqubm5wQ7DGGNCytq1a/eralpPx4VNssjNzWXNmq56UhpjjOmMiOzq+SirhjLGGBMASxbGGGN6ZMnCGGNMjyxZGGOM6ZElC2OMMT3yLFmIyEIRKReRz7rYLyLyexHZLiIbOqaRdPfdJCKfu6+bvIrRGGNMYLwsWTyDMwFLV2bjjP8/Hmc6yMfg8Pg09+LMBTAduNcdStoYY0yQeJYs3PFlDnRzyBxgkTo+AlJEZBTwZeAtVe2Y0estuk86Jtyowudvw8cLoGw9tLcFOyJjBr1gPpSXxdHTVRa727rafgwRuQ2nVEJOTk5nh5hQogrb3oCV/xdKC49sjxsKOV+C3PNgzHkwcgpEhs3zpMaEhGD+xUkn27Sb7cduVF0ALADIz8+3ERFDlSps+auTJPZugJQcuPr3MG4m7P4Ydr0HRe/Dtted42OTIeccyD0fxpwPo86w5GGMx4L5F1bM0XMhZwOl7vZZftvf6beoTP9pb4ctr8HK38C+T2HYWJjzKEy5ASKjnWOG5cIZNzjLh8pg1/tQ9K6TPD7/m7M9JtFJHmPOcxJI5tQj5xtj+kQwk8UrwB0ishinMbtaVctE5E3g330atS8DfhasII0H2tth88uw8j+gfCMMPwmufRxO/4fuSwjJo+D0650XQM2+I6WOXe/D8vuc7dEJMHq6kzhyz4fMaRAV4/3nMiaMeZYsROQvOCWEVBEpxunhFA2gqo8Dy4ArgO1APXCLu++AiPwbsNq91P2q2l1DuQkV7W2w6SUnSVRshhHjYe4CmHxd76qRkjKccydf56zXVrglj/ecn3//N2d7VPzRySPrTIiK7bvPZcwgEDaTH+Xn56uNOjtAtbfBZ0tg1X/A/q2QOgFm/AQmz4OISO/et67SSRodCWSf+8hPVBxkn+W2eZznLEfHeReHMQOYiKxV1fyejrNWQeOdtlb47EUnSVR+DmkT4fqFMOlab5NEh4QRMOka5wVQfwB2f+gkjqL34J1fAwqRMU7C6GjzyD4LYoZ4H58xIcRKFqbvtbXCp//rJIkDOyB9Esz8CUycAxEDaISZhoOw+6MjyWPvBtB2iIh2qqpyz3e6644+G2ISgh2tMZ4ItGRhycL0nbYW2PA/sOohOPgFZJzuJIlTrxpYSaIrjdVOV92id52qq9J1oG0QEeU0kuee53TVzTkbYpOCHa0xfcKShek/bS2w/i9Okqja5Tw0N/OncMoVoZEkutJU4/Ocx3vOg4LtrSCRkJl35DmPnHMgLjnY0ZpBqrm1nbqmVoYl9K7HnyUL473WZlj3HLz7W6je7TzfMPOnMOFykM6erQxxTbVQ/IlbbfU+lKyF9haQCOfBwI42j5xzIT4l2NGaEKaq1DS1Un6oiYqaJsprGqmo6Vg+sq28pomq+hbOHDOMF2//Uq/eyxq4jXdam6Dwz/Def0H1Hqd+/8r/hPGXhmeS6BCbCCdd5LwAmuvd5OH2tvpkAXz4CCAw8nTIvcCpuso5F4YMD2roZmBobWvnQF0z5T4JoPxQExW1zs/ymkYqap1k0NjSfsz5MVERpCfFkpYUy9jUBKaPHU56Uhzj0rxvU7OShQlcSyMU/slJEodKnF5DM++Gky8O7yQRqJYGKF5z5DmPPZ9AWxMgMHQ0pIx2hjJJyXHX3eXkLHto0AvtbVBTBlW7oWqP+3OX8wWnarfTRpWcCSljjv03Sck5rtJhfXPrUTf9Cvdbf7lfaaCyronObrlD46MPJ4H0pFjSk+NIS4wlPTn2yM+kOJLjopA+/luzkoXpOy0NULDISRI1ZTD6HJjzCIy70JKEr+h4GHuB8wInuZashV0fOF2Hq3bDF6vgUClHD3cm7k2rkxtWSg4MzbaHCDvT1up8aem4+R9OCruc5UMlThuTr8QM53c8Ks9pZzpUCpU7YMcKaKk76lCNTaYtOZuGIVkcih3F/qgMyiSd3W0j2N4ynKK6WCpqm6moaaK2ye99gKgIIdW90WelxJE3eihpSXFHEoKbHNKSYomN6oeu5CfIShamay0NsOZpeP93ULvXGfl11k9h7ExLEieitdm5kVXt7uRG597k1G9Y9qRRfomkY3mMk0yi44PzWbzk+3vq7Hd1zO9JIGlkp6W39uTRNCVkUq/RNLS00dDcRk1T65Fv/dUN1FWXowf3EF1bTEJ9CUOb95JJBdlSQbbsJ0kajgqvQeI5GJ1BbXwmTQnZMHQ0kSPGMCR9LMkZ40hJzSQicuB38LAGbtN7zXVHkkRduVP3PvOnR74xG2+1tUJN6bFJpKMKpbr42G/MCel+SaQjkbjVXwPxOZHWJuezVO2m/eBuWg4UoQd3QdUeIg/tIapuL+JTAmsngvq4dGpiR3IwZhQHojKoiMpgr6SzV9Ip0eEcaomksaWN+mYnIXQkhoaW7udEEYERCbFHVQUdXSUUw8iYRlLb9hFf5/tvs9vp3NFRreUrKr6Tf5McGOr+TEwfEF+6rBrKHL/mOlj9JHzwB6irgLEzYOYzTiOt6T+RUUduLJ3pri6+bD1sfs3ppeVrSKpfm0nO0aWUbp4bUVVa2vTwTbe+ufWom3B9c9vhG/SR5VZaGuuJrSslob6ExMZSkpv2MqxlL6mte0lrKyfVZ8i3CCBSI9jLcIo1jWIdT4meS7Gmuuup7NURtDQeuWXFR0cyJCaSOPdnfAzER8PwhBiyhx3ZPiQm6sgx0ZHEuz8TY6MOJ4ThCTFEBVQKyAKmdb6rsdrn36OjJORWiZUUQIPfEHdRcU6p8KiS0Jgj/yaJIwdU13MrWRinS+jqJ5wkUV/ptEXM/CmMOTfYkZkAqCpNre3Ot+mWNhqammmp2kv7wV1I9W4iDxUTU1tMbG0JQxpKSWwoJUqbj7pGbUQSFZEZ7ItIp5Q0ijWNXW0jKGodwfaWEVS3H1vNFU8jWbL/cDVNtrvcsS1Njv6m3UoklZHpHIgeSVVMBodiM6mLz6Q+IZPmhNG0J44kNi6WIR03dPem7n+Td27+EX3e0Ou5ppojycQ3kXRsq99/9PGRMUeSiX8iSclxqib7YNgcq4YyPWs85Hb3fNT51nPSxTDrbmeEVtNn2tvVuYm738brfb6hd1ZlUu+33NjDt/mGlrZOe9h0RWhnVGQtY6MrGRtZyejI/WTJfjK1nIz2clLb9hGrTUed0xiVRF3cKJri04lvqSKhvoSY5oNHHaORMbQnZSHDcpCUMYh/Q33SyP4ZEyxUNdcdrpZzEolfKaV239HHR0Q5PelScpyeiZfc26u3tWoo07XGamd+6w8fgcYqGH+ZU5LI7vH/iwlQdUMLK7dVsHzzPlZsKedQ47G9ZboTFx3BkJioo75Rx8dEkjIkhsxjvmlHEhcT6fONPKrHb+fdVrmoOiVMn2++cVW7iava7VR/JWfASfl+7SI5SGIGkQOo2iTkxCRA2inOqzMtDT7JpJNGf49ZshhMGqrg48fho/92EsaE2c7YTVld1MGa47Krso63N5fz9qZ9rC46QGu7MiIhhstOG8mEjET3Zh3VSV27zw09JpK4qEgiIoJYxSICCanOK+vM4MVhjhYdD6njnVcQWLIYDBoOwkePwUePQ1M1nHKlkyQy84IdWUhra1cKdx90EsTmfWwvrwVgQkYit80Yx8UTM8gbnUJkMG/8xvQRSxbhrP6A0x7x8R+huQYmXu1MOjRqSrAjC1m1Ta2s2lbB25v38c7WCg7UNRMVIZwzbgRfOzuHSyZmMHq4zYVhwo8li3BUV+m0R3yyAJprYdIcJ0mMnBzsyEJS8cF6lrulh493HqC5rZ2UIdFceEo6F09MZ8aENJLjooMdpjGesmTR0gAbng92FH1n/zbngbqWejhtLsz4F8iYFOyoQkp7u7K+uOpwgtiytwaAcWkJ3HxeLhefms6ZY4YF2C/fmPBgyaK5Dl69M9hR9CGBydc5SSL91GAHEzLqm1t57/P9LN9czvIt5eyvbSIyQsgfM4yfXzGRiyemMy4tMdhhGhM0niYLEbkc+B0QCTypqr/22z8GWAikAQeAG1W12N33G+BKnIc73wJ+oF48FBI/DH60qc8vGzTR8TYcdoD2VjeyfMs+lm8u5/3t+2lqbScpNoqZp6RxycQMZp2SRsoQGw3WGPAwWYhIJPAocClQDKwWkVdU1ffO/BCwSFWfFZGLgF8BXxeRLwHnAR0tse8BM4F3+jzQiEgYmtXnlzUDj6qysfQQb292EsSnJc4TxjnDh/BVt3H6rNzhxERZ9ZIx/rwsWUwHtqvqTgARWQzMAXyTxSTgR+7yCuAld1mBOCAGECAa8Ht80ZieNba08eGOSt7evI+/bymnrLoREZiWM4yfXH4Kl07M4OT0xNAbOsKYfuZlssgC9visFwNn+x2zHrgOp6pqLpAkIiNU9UMRWQGU4SSLR1R1s/8biMhtwG0AOTldDLpmBp2KmiZWbHEap9/9fD8NLW0MiYlkxvg0/vnSdC48NZ3URJsfwpjj4WWy6Oyrmn+bw13AIyJyM7AKKAFaReRkYCKQ7R73lojMUNVVR11MdQGwAJyxofowdhNCVJWt+2oO915at6cKVcgcGsf1Z2ZzyaQMzhk3PCQmmDFmoPIyWRQDo33Ws4FS3wNUtRSYByAiicB1qlrtlhg+UtVad9/rwDk4CcUYmlvb+fiLysMJovigMzHNGdlD+dElE7hkYgYTRyVZ9ZIxfcTLZLEaGC8iY3FKDF8Bvup7gIikAgdUtR34GU7PKIDdwLdF5Fc4JZSZwMMexmpCwMG6ZlZsdZLDqm37qW1qJS46gvNPTuWOC0/molPTSU+OC3aYxoQlz5KFqraKyB3AmzhdZxeq6kYRuR9Yo6qvALOAX4mI4pQavuee/gJwEfApTtXVG6r6qlexmoFJVdlRUcfyzft4e/M+1u46SLtCelIsV5+RySUT0znv5FTioq16yRiv2XwWZsA5WNfMog93sbSwmKLKegBOy0zm4okZXDIxncmZQ4M7KqsxYcTmszAhp6SqgSff3cniT/bQ0NLGBeNTufWCcVx8ajqZKcfO1GaM6T+WLEzQbdtXw+Mrd/DKOqf/w5y8LP5p5jgmZHQ9L7Qxpn9ZsjBBs6boAI+v3MHbm8uJj47kG+fmcusFY8myUoQxA44lC9Ov2tuVFVvLeXzlDlYXHWTYkGh+dMkEvnHuGIYl2DhMxgxUlixMv2hpa+fV9aU8vnIH2/bVkpUSzy+vnsQ/njWaITH239CYgc7+So2n6ptb+Z/Ve3jy3S8oqWrglIwk/uuGM7hqSibRNh+EMSHDkoXxxMG6Zp79sIhnPyjiYH0L03OH88C1k5l1Spo9VW1MCLJkYfqUf/fXSyZmcPuscZw5xubYMCaUWbIwfWLr3hr+uHIHr6w/0v31OzPHMd66vxoTFixZmBOypugAj72zg+VbyhkS43R//dYFY+0hOmPCjCULc9w6ur8+9s4O1uxyur/+86VO91ebhtSY8GTJwgSspa2dV9aV8sdVR7q/3nfNafxj/mjiY2wwP2PCmSUL06P65lYWf7KHp95zur+eOjKJh2/I48opo6z7qzGDhCUL06UDdc08+0ERz35YRFV9C9PHWvdXYwYrSxbmGMUH63ny3S/4n9VO99dLJ2XwnZknceaYYcEOzRgTJJYszGEd3V9fXl+KANdOzeKfZlj3V2OMJQsDrC46wOM+3V9v/lIut55v3V+NMUdYshik2tuVv28p57GVO1i76yDDE2Ks+6sxpkuWLAaZju6vj6/cwefl1v3VGBMYSxaDREf31yff3UlpdaN1fzXGHBdPk4WIXA78DogEnlTVX/vtHwMsBNKAA8CNqlrs7ssBngRGAwpcoapFXsYbjg7UNfPMB0Us+vBI99cH553OrAnW/dUYEzjPkoWIRAKPApcCxcBqEXlFVTf5HPYQsEhVnxWRi4BfAV939y0CHlTVt0QkEWj3KtZw1NH9dfHq3TS2tFv3V2PMCfGyZDEd2K6qOwFEZDEwB/BNFpOAH7nLK4CX3GMnAVGq+haAqtZ6GGdY2bL3EH9cuZNXfLq/fmfmOE5Ot+6vxpje8zJZZAF7fNaLgbP9jlkPXIdTVTUXSBKREcAEoEpElgBjgbeBu1W1zfdkEbkNuA0gJyfHi88QUlZuq+CmhZ9Y91djTJ/zMll0ViGufut3AY+IyM3AKqAEaHXjugCYCuwG/ge4GXjqqIupLgAWAOTn5/tfe9D504dFZCTH8uYPZ1j3V2NMn/KyG0wxTuN0h2yg1PcAVS1V1XmqOhX4ubut2j23UFV3qmorTvXUNA9jDXmVtU28s7WCa/OyLFEYY/qcl8liNTBeRMaKSAzwFeAV3wNEJFVEOmL4GU7PqI5zh4lImrt+EUe3dRg/r20oo7VdmTstK9ihGGPCkGfJwi0R3AG8CWwGnlfVjSJyv4hc4x42C9gqItuADOBB99w2nCqq5SLyKU6V1hNexRoOlhSWMGlUMqeOTA52KMaYMOTpcxaqugxY5rftFz7LLwAvdHHuW8AUL+MLFzsqalm/p4p7rpwY7FCMMWHKHt0NA0sLSogQuOaMzGCHYowJU5YsQlx7u7K0sITzx6eRnhwX7HCMMWHKkkWI+6ToACVVDVxnDdvGGA9ZsghxSwtKSIiJ5LJJI4MdijEmjFmyCGGNLW0s+7SMyyePsuHFjTGesmQRwt7evI+aplargjLGeM6SRQhbUlDCqKFxnDNuRLBDMcaEOUsWIWp/bRMrt1UwJy+LiAibl8IY4y1LFiHq1fWltLUr86wKyhjTDyxZhKglBSVMzkpmQobNU2GM8Z4lixC0vbyGT0uqmTs1O9ihGGMGCUsWIWhJQQmREWLDexhj+o0lixDT3q68VFjCjPGppCXFBjscY8wgYckixHz0RSWl1Y3MnWZVUMaY/mPJIsQsLSghMTaKyyZlBDsUY8wgYskihDQ0t/H6Z3uZPXkkcdE2vIcxpv9Ysgghf9u0l9qmVuZZFZQxpp/1mCxExL7CDhBLC0vISonn7LHDgx2KMWaQCaRksV1E/kNEJnkejelSRU0T736+nzl5mTa8hzGm3wWSLKYA24AnReQjEblNRJI9jsv4ecWG9zDGBFGPyUJVa1T1CVX9EvAT4F6gTESeFZGTuztXRC4Xka0isl1E7u5k/xgRWS4iG0TkHRHJ9tufLCIlIvLIcX6usLOkoJgp2UM5Od2G9zDG9L+A2ixE5BoRWQr8DvhPYBzwKrCsu/OAR4HZwCRgfidVWQ8Bi1R1CnA/8Cu//f8GrAzws4Stbftq2Fh6iLlTrVRhjAmOqACO+RxYAfyHqn7gs/0FEZnRzXnTge2quhNARBYDc4BNPsdMAn7kLq8AXurYISJnAhnAG0B+AHGGrY7hPa624T2MMUESUJuFqt7qlygAUNU7uzkvC9jjs17sbvO1HrjOXZ4LJInICBGJwCnB/Et3gbntJ2tEZE1FRUVPnyMktbnDe8yakEZqog3vYYwJjkCSRauIfE9E/ltEFna8Ajivsy476rd+FzBTRAqBmUAJ0Ap8F1imqnvohqouUNV8Vc1PS0sLIKTQ89HOSvYeamSuNWwbY4IokGqoPwFbgC/jtCt8DdgcwHnFwGif9Wyg1PcAVS0F5gGISCJwnapWi8i5wAUi8l0gEYgRkVpVPaaRPNwtKSghKTaKSyba8B7GmOAJpGRxsqr+H6BOVZ8FrgROD+C81cB4ERkrIjHAV4BXfA8QkVS3ygngZ8BCAFX9mqrmqGouTulj0WBMFPXNrbzxWRlXnD7KhvcwxgRVIMmixf1ZJSKTgaFAbk8nqWorcAfwJk5J5HlV3Sgi94vINe5hs4CtIrINpzH7weMLP7z9beM+6prb7NkKY0zQBVINtUBEhgH34JQMEoH/E8jFVXUZft1rVfUXPssvAC/0cI1ngGcCeb9ws8Qd3uOsXBvewxgTXN0mC7eK6JCqHgRW4TxfYfpB+aFG3vu8gu/OOtmG9zDGBF231VCq2o5TlWT62cvrSmlXrBeUMWZACKTN4i0RuUtERovI8I6X55ENcksKSzhjdAonpSUGOxRjjAmozeKb7s/v+WxTrErKM1v2HmJz2SHuu+a0YIdijDFAAMlCVcf2RyDmiKUFJUTZ8B7GmAGkx2QhIt/obLuqLur7cExbu/LSuhJmnZLO8ISYYIdjjDFAYNVQZ/ksxwEXAwWAJQsPfLBjP/sONXHv1dawbYwZOAKphvq+77qIDMUZAsR4YGlBCUlxUVx0anqwQzHGmMMC6Q3lrx4Y39eBGKhrauX1z/Zy1ZRMG97DGDOgBNJm8SpHRouNwJmD4nkvgxqs3ty4l4YWG97DGDPwBNJm8ZDPciuwS1WLPYpnUFtaWMLo4fHkjxkW7FCMMeYogSSL3UCZqjYCiEi8iOSqapGnkQ0ye6sbeX/7fu648GREbHgPY8zAEkibxf8C7T7rbe4204deXlfiDu+RHexQjDHmGIEkiyhVbe5YcZftAYA+trSwhKk5KYxNTQh2KMYYc4xAkkWFz/wTiMgcYL93IQ0+m0oPsWVvDfOmWsO2MWZgCqTN4jvAcyLyiLteDHT6VLfpnSUFxURHCldNseE9jDEDUyAP5e0AznHnyBZVrfE+rMGjta2dl9eXcuEp6Qyz4T2MMQNUj9VQIvLvIpKiqrWqWiMiw0Tkgf4IbjB4f0clFTVN9myFMWZAC6TNYraqVnWsuLPmXeFdSIPLkoJihsZHc6EN72GMGcACSRaRIhLbsSIi8UBsN8ebANU2tfLmxr1cNWUUsVE2vIcxZuAKJFn8GVguIreKyK3AW8CzgVxcRC4Xka0isl1E7u5k/xgRWS4iG0TkHRHJdrfniciHIrLR3XfD8XyoUPHGZ3tpbGm3KihjzIAXSAP3b0RkA3AJIMAbwJiezhORSOBR4FKcHlSrReQVVd3kc9hDwCJVfVZELgJ+BXwdZ7DCb6jq5yKSCawVkTd9q8PCwdLCYsaMGMK0HBvewxgzsAU66uxenKe4r8OZz2JzAOdMB7ar6k73Qb7FwBy/YyYBy93lFR37VXWbqn7uLpcC5UBagLGGhLLqBj7YUcm1eVk2vIcxZsDrMlmIyAQR+YWIbAYeAfbgdJ29UFUf6eo8H1nuOR2K3W2+1uMkIIC5QJKIjPCLYzrOE+M7OonxNhFZIyJrKioqAghp4HipsBRVrArKGBMSuitZbMEpRVytquer6h9wxoUKVGdfl9Vv/S5gpogUAjNDEfu5AAAVnElEQVSBEpyRbZ0LiIzCmWjpFlVt9zsXVV2gqvmqmp+WFjoFD1VlSUExZ44ZxpgRNryHMWbg6y5ZXIdT/bRCRJ4QkYvpPAF0pRgY7bOeDZT6HqCqpao6T1WnAj93t1UDiEgy8FfgHlX96Djed8DbWHqIz8trmWvDexhjQkSXyUJVl6rqDcCpwDvAj4AMEXlMRC4L4NqrgfEiMlZEYoCvAK/4HiAiqSLSEcPPgIXu9hhgKU7jd9iNcLukoISYyAiumjIq2KEYY0xAemzgVtU6VX1OVa/CKR2sA47pBtvJea3AHcCbOA3iz6vqRhG532dgwlnAVhHZBmQAD7rb/xGYAdwsIuvcV95xfrYBqbWtnVfWl3LRqemkDLHhPYwxoSGQgQQPU9UDwB/dVyDHLwOW+W37hc/yC8ALnZz3Z5znO8LOu9v3s7+2ibnWsG2MCSGBdp01fWRJQQkpQ6K58BQb3sMYEzosWfSjmsYW/rZxL1dPySQmyn71xpjQYXesfvT6Z3tpam23KihjTMixZNGPlhQUMzY1gamjU4IdijHGHBdLFv2kpKqBj3YeYO5UG97DGBN6LFn0k5cKSwDsQTxjTEiyZNEPVJWlhSWclTuM0cOHBDscY4w5bpYs+sGnJdVsL69l7tTsYIdijDG9YsmiHywpKCEmKoIrT7fhPYwxocmShcda2tp5dX0pl0xMZ+iQ6GCHY4wxvWLJwmOrtlVQWddsVVDGmJBmycJjSwpLGJ4Qw8wJoTPfhjHG+LNk4aFDjS28tWkfV08ZZcN7GGNCmt3BPPT6p2U0t7Yzd5pVQRljQpslCw+9WFDCuNQEzsgeGuxQjDHmhFiy8MieA/V88sUB5k2z4T2MMaHPkoVHXl7nDO8xJ8+G9zDGhD5LFh5QVZYUlDB97HAb3sMYExYsWXhgfXE1O/fXMc8GDTTGhAlLFh5YWlBMTFQEV0yx4T2MMeHB02QhIpeLyFYR2S4id3eyf4yILBeRDSLyjohk++y7SUQ+d183eRlnX2pubefVDWVcOimD5Dgb3sMYEx48SxYiEgk8CswGJgHzRWSS32EPAYtUdQpwP/Ar99zhwL3A2cB04F4RGeZVrH1p5bYKDtQ1WxWUMSaseFmymA5sV9WdqtoMLAbm+B0zCVjuLq/w2f9l4C1VPaCqB4G3gMs9jLXPLC0sZkRCDDNseA9jTBjxMllkAXt81ovdbb7WA9e5y3OBJBEZEeC5iMhtIrJGRNZUVFT0WeC9VV3fwtuby7n6jEyiI605yBgTPry8o3X2JJr6rd8FzBSRQmAmUAK0BnguqrpAVfNVNT8tLfjf5P/qDu8xb5pVQRljwkuUh9cuBkb7rGcDpb4HqGopMA9ARBKB61S1WkSKgVl+577jYax9YmlhMSelJXB6lg3vYYwJL16WLFYD40VkrIjEAF8BXvE9QERSRaQjhp8BC93lN4HLRGSY27B9mbttwNpzoJ7VRQeZNy3bhvcwxoQdz5KFqrYCd+Dc5DcDz6vqRhG5X0SucQ+bBWwVkW1ABvCge+4B4N9wEs5q4H5324C1tNAZ3uNa6wVljAlDonpMU0BIys/P1zVr1gTlvVWVCx96h5FD41h827lBicEYY3pDRNaqan5Px1mXnT5QuKeKosp65tm8FcaYMGXJog8sLSghNiqC2ZNHBjsUY4zxhCWLE+QM71HKZaeNJMmG9zDGhClLFidoxdZyqupbbHgPY0xYs2RxgpYWlJCaGMMF41ODHYoxxnjGksUJqKpv5u9byrnmjCyibHgPY0wYszvcCXhtQxnNbTa8hzEm/FmyOAFLC0uYkJHIaZnJwQ7FGGM8Zcmil3ZV1rF210HmTrXhPYwx4c+SRS8tKShBBK6dmhnsUIwxxnOWLHpBVXlpXQnnjhvBqKHxwQ7HGGM8Z8miFwp2H2SXDe9hjBlELFn0wosFJcRFR3C5De9hjBkkLFkcp6bWNv66oYwvnzaSxFgv544yxpiBw5LFcVqxpZzqhhargjLGDCqWLI7TkoIS0pJiOe+kEcEOxRhj+o0li+NwsK6ZFVvLmXNGpg3vYYwZVOyOdxxe21BKS5sy14b3MMYMMpYsjsOSwhJOHZnEpFE2vIcxZnCx7jwB2llRS+HuKn42+1Qb3sMYj7W0tFBcXExjY2OwQwkbcXFxZGdnEx3du0naPE0WInI58DsgEnhSVX/ttz8HeBZIcY+5W1WXiUg08CQwzY1xkar+ystYe/JSoTO8x5w8q4IyxmvFxcUkJSWRm5trX876gKpSWVlJcXExY8eO7dU1PKuGEpFI4FFgNjAJmC8ik/wOuwd4XlWnAl8B/tvd/g9ArKqeDpwJ/JOI5HoVa09UlaXrSjjvpFRGDo0LVhjGDBqNjY2MGDHCEkUfERFGjBhxQiU1L9sspgPbVXWnqjYDi4E5fsco0NEAMBQo9dmeICJRQDzQDBzyMNZurdl1kD0HGmzeCmP6kSWKvnWiv08vk0UWsMdnvdjd5uuXwI0iUgwsA77vbn8BqAPKgN3AQ6p6wP8NROQ2EVkjImsqKir6OPwjlhQUEx8dyZdPs+E9jDGDk5fJorM0pn7r84FnVDUbuAL4k4hE4JRK2oBMYCzwYxEZd8zFVBeoar6q5qelpfVt9K7GljZe21DG5ZNHkmDDexgzKFRWVpKXl0deXh4jR44kKyvr8Hpzc3NA17jlllvYunWrx5H2Hy/vfsXAaJ/1bI5UM3W4FbgcQFU/FJE4IBX4KvCGqrYA5SLyPpAP7PQw3k79fUs5NY2tVgVlzCAyYsQI1q1bB8Avf/lLEhMTueuuu446RlVRVSIiOv/O/fTTT3seZ3/yMlmsBsaLyFigBKcB+6t+x+wGLgaeEZGJQBxQ4W6/SET+DAwBzgEe9jDWLi0pKCYjOZYvnZQajLc3ZtC779WNbCrt2ybLSZnJ3Hv1acd93vbt27n22ms5//zz+fjjj3nttde47777KCgooKGhgRtuuIFf/OIXAJx//vk88sgjTJ48mdTUVL7zne/w+uuvM2TIEF5++WXS09P79DN5zbNqKFVtBe4A3gQ24/R62igi94vINe5hPwa+LSLrgb8AN6uq4vSiSgQ+w0k6T6vqBq9i7UplbRPvbK1gTl4WkRHW2GaMgU2bNnHrrbdSWFhIVlYWv/71r1mzZg3r16/nrbfeYtOmTcecU11dzcyZM1m/fj3nnnsuCxcuDELkJ8bTSnhVXYbTcO277Rc+y5uA8zo5rxan+2xQvbahjNZ2Ze5Uq4IyJlh6UwLw0kknncRZZ511eP0vf/kLTz31FK2trZSWlrJp0yYmTTr6KYH4+Hhmz54NwJlnnsm7777brzH3BWux7caSwhImjkpmog3vYYxxJSQkHF7+/PPP+d3vfscnn3xCSkoKN954Y6fPMsTExBxejoyMpLW1tV9i7Us2NlQXdlTUsn5PFfOsVGGM6cKhQ4dISkoiOTmZsrIy3nzzzWCH5BkrWXRhaUEJEQJz8jKDHYoxZoCaNm0akyZNYvLkyYwbN47zzjumVj1siNOeHPry8/N1zZo1fXKt9nblgt+s4KT0RBZ9c3qfXNMYE7jNmzczceLEYIcRdjr7vYrIWlXN7+lcq4bqxCdFByiparAqKGOMcVmy6MTSghKGxERy2WkZwQ7FGGMGBEsWfhpb2lj2qTO8x5AYa9IxxhiwZHGMtzfvo6apleumZQc7FGOMGTAsWfhZUlDCyOQ4zhk3ItihGGPMgGHJwsf+2iZWbqtgztRMG97DGGN8WLLw8er6UtralXlTrQrKmMFs1qxZxzxg9/DDD/Pd7363y3MSExMBKC0t5frrr+/yuj118X/44Yepr68/vH7FFVdQVVUVaOiesWThY0lBCadlJnPKyKRgh2KMCaL58+ezePHio7YtXryY+fPn93huZmYmL7zwQq/f2z9ZLFu2jJSUlF5fr69Ydx/X9vIaPi2p5p4r7UEgYwaU1++GvZ/27TVHng6zf93l7uuvv5577rmHpqYmYmNjKSoqorS0lLy8PC6++GIOHjxIS0sLDzzwAHPmHD1bdFFREVdddRWfffYZDQ0N3HLLLWzatImJEyfS0NBw+Ljbb7+d1atX09DQwPXXX899993H73//e0pLS7nwwgtJTU1lxYoV5ObmsmbNGlJTU/ntb397eMTab33rW/zwhz+kqKiI2bNnc/755/PBBx+QlZXFyy+/THx8fJ/+yqxk4VriDu9xjQ3vYcygN2LECKZPn84bb7wBOKWKG264gfj4eJYuXUpBQQErVqzgxz/+Md2NgvHYY48xZMgQNmzYwM9//nPWrl17eN+DDz7ImjVr2LBhAytXrmTDhg3ceeedZGZmsmLFClasWHHUtdauXcvTTz/Nxx9/zEcffcQTTzxBYWEh4Axo+L3vfY+NGzeSkpLCiy++2Oe/EytZ4Azv8VJhCTMmpJGeFBfscIwxvropAXipoypqzpw5LF68mIULF6Kq/Ou//iurVq0iIiKCkpIS9u3bx8iRIzu9xqpVq7jzzjsBmDJlClOmTDm87/nnn2fBggW0trZSVlbGpk2bjtrv77333mPu3LmHR72dN28e7777Ltdccw1jx44lLy8PcIZALyoq6qPfwhFWsgA++qKS0upGm7fCGHPYtddey/Llyw/Pgjdt2jSee+45KioqWLt2LevWrSMjI6PTIcl9iRzbs/KLL77goYceYvny5WzYsIErr7yyx+t0V4KJjY09vOzVEOiWLHCG90iMjeKySZ1/OzDGDD6JiYnMmjWLb37zm4cbtqurq0lPTyc6OpoVK1awa9eubq8xY8YMnnvuOQA+++wzNmxwJvw8dOgQCQkJDB06lH379vH6668fPicpKYmamppOr/XSSy9RX19PXV0dS5cu5YILLuirj9ujQV8N1dDcxuuf7WX25JHEx0QGOxxjzAAyf/585s2bd7hn1Ne+9jWuvvpq8vPzycvL49RTT+32/Ntvv51bbrmFKVOmkJeXx/TpzijWZ5xxBlOnTuW00047Zmjz2267jdmzZzNq1Kij2i2mTZvGzTfffPga3/rWt5g6daonVU6dGfRDlO871MgDf93MjWfncLY9tW3MgGBDlHvjRIYoH/Qli4zkOP4wf2qwwzDGmAHN0zYLEblcRLaKyHYRubuT/TkiskJECkVkg4hc4bNvioh8KCIbReRTEbFuSsYYEySelSxEJBJ4FLgUKAZWi8grqrrJ57B7gOdV9TERmQQsA3JFJAr4M/B1VV0vIiOAFq9iNcYMPKraaU8i0zsn2uTgZcliOrBdVXeqajOwGJjjd4wCye7yUKDUXb4M2KCq6wFUtVJV2zyM1RgzgMTFxVFZWXnCNzjjUFUqKyuJi+t9BY2XbRZZwB6f9WLgbL9jfgn8TUS+DyQAl7jbJwAqIm8CacBiVf2N/xuIyG3AbQA5OTl9GrwxJniys7MpLi6moqIi2KGEjbi4OLKzez9IqpfJorPyo//XhPnAM6r6nyJyLvAnEZnsxnU+cBZQDyx3W+yXH3Ux1QXAAnB6Q/X1BzDGBEd0dDRjx44NdhjGh5fVUMXAaJ/1bI5UM3W4FXgeQFU/BOKAVPfclaq6X1XrcdoypnkYqzHGmG54mSxWA+NFZKyIxABfAV7xO2Y3cDGAiEzESRYVwJvAFBEZ4jZ2zwQ2YYwxJig8q4ZS1VYRuQPnxh8JLFTVjSJyP7BGVV8Bfgw8ISI/wqmiulmdFq2DIvJbnISjwDJV/atXsRpjjOle2DzBLSIVQPcDtXQvFdjfR+EEU7h8DrDPMlCFy2cJl88BJ/ZZxqhqWk8HhU2yOFEisiaQR94HunD5HGCfZaAKl88SLp8D+uez2KizxhhjemTJwhhjTI8sWRyxINgB9JFw+Rxgn2WgCpfPEi6fA/rhs1ibhTHGmB5ZycIYY0yPLFkYY4zp0aBPFj3NuREqRGShiJSLyGfBjuVEichod56Tze58Jj8Idky9ISJxIvKJiKx3P8d9wY7pRIlIpDv/zGvBjuVEiEiRO0/OOhE5/ik2BxARSRGRF0Rki/s3c64n7zOY2yzcOTe24TPnBjDfb86NkCAiM4BaYJGqTg52PCdCREYBo1S1QESSgLXAtaH27yLOZAwJqlorItHAe8APVPWjIIfWayLyz0A+kKyqVwU7nt4SkSIgX1VD/qE8EXkWeFdVn3SHVhqiqlV9/T6DvWQRyJwbIUFVVwEHgh1HX1DVMlUtcJdrgM04Q96HFHXUuqvR7itkv52JSDZwJfBksGMxDhFJBmYATwGoarMXiQIsWXQ250bI3ZTCmYjkAlOBj4MbSe+41TbrgHLgLVUNyc/hehj4CdAe7ED6gOLMpbPWnRcnVI3DGXz1abd68EkRSfDijQZ7sghkzg0TJCKSCLwI/FBVDwU7nt5Q1TZVzcMZon+6O19LyBGRq4ByVV0b7Fj6yHmqOg2YDXzPrcYNRVE40zc8pqpTgTrAk7bXwZ4sAplzwwSBW8f/IvCcqi4Jdjwnyq0aeAe4PMih9NZ5wDVuXf9i4CIR+XNwQ+o9VS11f5YDS3GqpENRMVDsU2J9AY/m/hnsySKQOTdMP3Mbhp8CNqvqb4MdT2+JSJqIpLjL8TjTBm8JblS9o6o/U9VsVc3F+Tv5u6reGOSwekVEEtyOE7hVNpcBIdmLUFX3AntE5BR308V4NPePl9OqDnhdzbkR5LB6RUT+AswCUkWkGLhXVZ8KblS9dh7wdeBTt74f4F9VdVkQY+qNUcCzbq+7COB5VQ3pLqdhIgNY6nwnIQr4f6r6RnBDOiHfB55zv/DuBG7x4k0GdddZY4wxgRns1VDGGGMCYMnCGGNMjyxZGGOM6ZElC2OMMT2yZGGMMaZHliyMOQ4i0uaOVNrx6rOnZUUkNxxGDTbhaVA/Z2FMLzS4w3cYM6hYycKYPuDOj/B/3fkrPhGRk93tY0RkuYhscH/muNszRGSpO9fFehH5knupSBF5wp3/4m/uk9/GBJ0lC2OOT7xfNdQNPvsOqep04BGcEVpxlxep6hTgOeD37vbfAytV9QycsXw6Rg4YDzyqqqcBVcB1Hn8eYwJiT3AbcxxEpFZVEzvZXgRcpKo73UEQ96rqCBHZjzORU4u7vUxVU0WkAshW1Safa+TiDGM+3l3/KRCtqg94/8mM6Z6VLIzpO9rFclfHdKbJZ7kNa1c0A4QlC2P6zg0+Pz90lz/AGaUV4Gs4U6sCLAduh8MTJCX3V5DG9IZ9azHm+MT7jIQL8IaqdnSfjRWRj3G+hM13t90JLBSRf8GZ0axjRNAfAAtE5FacEsTtQJnn0RvTS9ZmYUwfcNss8lV1f7BjMcYLVg1ljDGmR1ayMMYY0yMrWRhjjOmRJQtjjDE9smRhjDGmR5YsjDHG9MiShTHGmB79f/pdJfReMGyQAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_history(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      1740\n",
      "           1       1.00      1.00      1.00       344\n",
      "\n",
      "   micro avg       1.00      1.00      1.00      2084\n",
      "   macro avg       1.00      1.00      1.00      2084\n",
      "weighted avg       1.00      1.00      1.00      2084\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model(model, train_sequences, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99       192\n",
      "           1       1.00      0.95      0.97        39\n",
      "\n",
      "   micro avg       0.99      0.99      0.99       231\n",
      "   macro avg       0.99      0.97      0.98       231\n",
      "weighted avg       0.99      0.99      0.99       231\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model(model, valid_sequences, valid_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       480\n",
      "           1       0.95      0.93      0.94        98\n",
      "\n",
      "   micro avg       0.98      0.98      0.98       578\n",
      "   macro avg       0.97      0.96      0.96       578\n",
      "weighted avg       0.98      0.98      0.98       578\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model(model, test_sequences, test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_model(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}