NLP_Exercise_2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "\n",
    "#from nltk.tokenize import word_tokenize\n",
    "\n",
    "#from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split  \n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.utils.fixes import signature\n",
    "from sklearn.metrics import classification_report, accuracy_score, precision_recall_curve, precision_recall_fscore_support\n",
    "from sklearn.decomposition import PCA, TruncatedSVD\n",
    "from sklearn.feature_selection import mutual_info_classif\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.dummy import DummyClassifier\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn.model_selection import learning_curve\n",
    "from sklearn.model_selection import ShuffleSplit\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import operator\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import numpy as np\n",
    "from numpy import array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using all 10 data folders for training and validation, splitting them later at random"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## You can use 10-fold cross validation experiments for HYPER PARAMETER TUNING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#directory_path = r\"C:\\Users\\Κωστας\\Desktop\\SpamHam_NLP\\lingspam_public\\lemm\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "directory_path = \"/media/spiros/Data/lingspam_public/bare/\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load txt files and create a mail list (inputs) and a spam/ham list (outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#spam_ham_tuple_list = list()\n",
    "mails = list()\n",
    "spams = list()\n",
    "for folder in os.listdir(directory_path):\n",
    "    for file in os.listdir(os.path.join(directory_path, folder)):\n",
    "        if file.endswith(\".txt\"):\n",
    "            with open(os.path.join(directory_path, folder, file), encoding='utf8') as f:\n",
    "                mail = f.read().strip()\n",
    "                # Clean each mail\n",
    "                # Remove all the special characters\n",
    "                mail = re.sub(r'\\W', ' ', mail)\n",
    "\n",
    "                # Remove all single characters\n",
    "                mail = re.sub(r'\\s+[a-zA-Z]\\s+', ' ', mail)\n",
    "\n",
    "                # Substituting multiple spaces with single space\n",
    "                mail = re.sub(r'\\s+', ' ', mail, flags=re.I)\n",
    "\n",
    "                # Convert to Lowercase\n",
    "                mail = mail.lower()\n",
    "                \n",
    "                mails.append(mail)\n",
    "                \n",
    "                if file.startswith(\"spm\"):\n",
    "                    #spam_ham_tuple_list.append((mail,1))\n",
    "                    spams.append(1)\n",
    "                else:\n",
    "                    #spam_ham_tuple_list.append((mail,0))\n",
    "                    spams.append(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mail_word_sequences = list()\n",
    "# for mail, spam in spam_ham_tuple_list:\n",
    "#     mail_word_sequences.append((word_tokenize(mail), spam))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mails = list()\n",
    "# spams = list()\n",
    "# for mail, spam in spam_ham_tuple_list:\n",
    "#     mails.append(mail)\n",
    "#     spams.append(spam)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Much more ham than spam mails, however if the recall turns out to be high, I suggest not adding more copies of spam mails in order to achieve a 50-50 distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ham mails count: 2412\n",
      "Spam mails count: 481\n"
     ]
    }
   ],
   "source": [
    "unique, counts = np.unique(spams, return_counts=True)\n",
    "print(\"Ham mails count: \"+str(counts[0]))\n",
    "print(\"Spam mails count: \"+str(counts[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = array(spams)\n",
    "#dictionary = dict(zip(X, y))\n",
    "#dictionary=DictVectorizer()\n",
    "#dictionary.get_feature_names()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train Test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(mails, y, test_size=0.2, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TF-IDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### From https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.TfidfVectorizer.html:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### max_df : float in range [0.0, 1.0] or int (default=1.0) When building the vocabulary ignore terms that have a document frequency strictly higher than the given threshold (corpus-specific stop words). If float, the parameter represents a proportion of documents, integer absolute counts."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### min_df : float in range [0.0, 1.0] or int (default=1) When building the vocabulary ignore terms that have a document frequency strictly lower than the given threshold. This value is also called cut-off in the literature. If float, the parameter represents a proportion of documents, integer absolute counts."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### max_features : int or None (default=None) If not None, build a vocabulary that only consider the top max_features ordered by term frequency across the corpus."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ngram_range : tuple (min_n, max_n) (default=(1, 1)) The lower and upper boundary of the range of n-values for different n-grams to be extracted. All values of n such that min_n <= n <= max_n will be used."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidfconverter = TfidfVectorizer(min_df=10, max_df=0.8, stop_words='english', ngram_range=(1,3)\n",
    "                                 , max_features=100\n",
    "                                )  \n",
    "X_train = tfidfconverter.fit_transform(X_train).toarray()\n",
    "\n",
    "X_test = tfidfconverter.transform(X_test).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Input and output feature vectors dimensions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2314, 1000)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(579, 1000)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2314,)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(579,)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Information gain / Mutual Information feature selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mutual_info_classif_custom(X, y):\n",
    "    return mutual_info_classif(X, y, discrete_features=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train = SelectKBest(mutual_info_classif_custom, k=1000).fit_transform(X_train, y_train)\n",
    "# X_test = SelectKBest(mutual_info_classif_custom, k=1000).fit_transform(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# a=0.88\n",
    "# #takes an argument \"a\" and cuts the a% percentage of the features with the lower Information Gain\n",
    "# def InformationGain(a,X,y):\n",
    "#     inf_gain = mutual_info_classif(X, y, discrete_features=True)\n",
    "#     print(len(inf_gain))\n",
    "#     #inf_gain.shape\n",
    "#     res = dict(zip(tfidfconverter.get_feature_names(),inf_gain))\n",
    "#     res_sort = sorted(res.items(), key=operator.itemgetter(1), reverse=True)\n",
    "#     y=len(res)\n",
    "#     if a>=0 and a<=1:\n",
    "#         while (1-a)*y<=len(res_sort):\n",
    "#             res_sort.pop()\n",
    "#     else :\n",
    "#         print(\"Not acceptable argument for variable 'a'\")\n",
    "        \n",
    "#     #print(res_sort)\n",
    "#     return res_sort"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'X_train' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-d2ba684acd0f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mX_train\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'X_train' is not defined"
     ]
    }
   ],
   "source": [
    "#X_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PCA Dimensonality reduction (Does nothing for now)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Cumulative explained variance')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8FeXZ//HPlR0IJECQfVMQ2Vwj7lXcarUVrVrF+litxS5qtXZ59Ofzq4r9tVprrX1q3ahaW/elLSpirbvWCgHZBWQnrIFAgISs5/r9MUM8RkgGzMlJcr7v1+u8zsw9c2auycC5zsw9932buyMiIgKQluwARESk9VBSEBGRekoKIiJST0lBRETqKSmIiEg9JQUREamnpCAiIvWUFEREpJ6SgoiI1MtIdgB7q6CgwAcNGpTsMERE2pQZM2ZscvceTa3X5pLCoEGDKCoqSnYYIiJtipmtjLKebh+JiEg9JQUREamnpCAiIvWUFEREpJ6SgoiI1EtYUjCzh81so5nN28NyM7Pfm9kSM5tjZocnKhYREYkmkVcKjwJnNLL8K8DQ8HUlcF8CYxERkQgS1k7B3d8xs0GNrDIOeMyD8UD/Y2b5Ztbb3dclKiYRkWSpro1RXlXLzpo6KmvqqKqNUVlTR2VNjKra+PdPl1XXxog5xNyJxZxThvfkkP75CY0zmY3X+gKr4+aLw7LPJQUzu5LgaoIBAwa0SHAiIrtU18bYurOasooatu6sYWtFDVsrqinbWUN5VR07qmrYUVXHjqpayqtq2VFZy/Zd0+Grujb2hePYr0tOu04Kkbn7g8CDAIWFhZ7kcESkDXN3ynbWsGlHFSXbq9m0o6r+VVpeQ9nO6vBLv4ayncGXf3l1XaPb7JCZTqfsDDrnZNApO53c7Az65ncgNzud3JwMcrMzyc0O1umYlU52Rjo5mWlkZ6aTnZFGTtx7/HRWehrpaUaagZm1yN8nmUlhDdA/br5fWCYistdq62Js3F7F+m2VrC+rZF1ZJSXbqz7zpb9pezWby6uoqfv8b8v0NKNrx0zyO2aR3yGTPvk5DO/dhfyOmeR3yCS/YyZ54bKgLIsuHTLIzc4gI739PMiZzKQwGbjazJ4CjgLKVJ8gIrtTF3M2bKtkdWkF68oq4774d7I+nC/ZXkWswXd9ZrpRkJtNQW42PXKzGd6rCwWds+neKYsenbPrlxXkZtG1YxZpaS3za7w1S1hSMLMngZOAAjMrBm4GMgHc/X5gCnAmsASoAC5PVCwi0rq5OyXbq1i9ZSfFWyoo3rKT1aXh+5YK1m7d+blf951zMuidl0PPLjkM69WZXnkd6NUlh955OfTKy6FXlxzyO2a22G2X9iKRTx+Nb2K5A1clav8i0rq4O5t2VLN8UzkrNpWzbFM5yzftYPmmclZurqCqQUVsQW4W/bp2ZHTfPM4c3Zv+XTvSr2sH+nYNvvw7ZbeJKtE2R39VEWlWVbV1LN1Yzicbt7N8U/mnr5JytlfV1q+XmW4M6NaRwQW5fGloD/p360j/bh3o37Ujfbt2oGOWvp6SQX91EdknNXUxlm8qZ/GG7SzesIPF67ezeON2Vm6uoC68uW8GffM7MLigE+ce3pfBBZ0YXNCJ/Qty6ZOf064qaNsLJQURaVJpeTXz15Yxf+025q/dxqL121i+qbz+Pn+awaDunRjaM5ezRvfmwJ6dGdozl0HdO5GTmZ7k6GVvKCmISD13Z21ZJfPX7EoAwfu6ssr6dfrmd2B4786cMrwnw8Iv/wN65OrLv51QUhBJYaXl1cxevZWPVm3ho9VbmbemjC0VNUDw63//HrmMGdyNkX26MLJPHiP7dCG/Y1aSo5ZEUlIQSRHVtTEWrNvGrFVbmLV6Kx+t3srKzRVAkAAO7NmZ00f0YlTfLozok8fw3p1V2ZuCdMZF2qmyihqmryhl+opSpq0oZf6abVTXBY999uySzaH98xk/ZgCH9s9ndN88PeIpgJKCSLuxvqySaStKmb48SASLNmzHHbLS0zi4Xx6XHTeIw/rnc+iAfHrndUh2uNJKKSmItFEbt1fywdLNvPfJJj5cXsqq0uBWUMesdI4Y2JWzRvfmyMHdOLR/viqBJTIlBZE2oryqlmnLS3lvySbe+2QTizZsByCvQyZHDe7GpccMZMzgbozo3UXP/8s+U1IQaaXqYs7s4q28u3gT7y/ZxMxVW6iNOVkZaYwZ1I1zDuvLcUO6M7JPHunqyE2aiZKCSCuytaKatxeX8ObCjby9uIQtFTWYwei+eUz40v4cP6SAIwZ21e0gSRglBZEkcncWrt/OGws38ubCjcxctYWYQ9eOmZw0bD/GHrQfJwwpoGsntQ2QlqGkINLC6mLO9BWlTJ23nn/OX8/asLXwyD5duGrsEMYetB+H9MvXLSFJCiUFkRZQVVvHv5du5tV563ltwQY2l1eTlZHGl4YW8MNThjL2oP3o2SUn2WGKKCmIJEplTR1vLtzIK/PW88bCjeyoqiU3O4OxB+3HGSN7cdKwHmowJq2O/kWKNKOauhjvLdnEi7PW8s8FG9hRVUu3TlmcNbo3Z4zqxbFDupOdoUpiab2aTApm1hP4JdDH3b9iZiOAY9z9TwmPTqQNiIV1BJNnr2XK3HVsqaihc04GZ47uxdmH9OXo/bup3YC0GVGuFB4FHgFuCucXA08DSgqS0hZv2M5zM4qZPGst67dV0iEznVNH9OTsQ/rwpQMLdEUgbVKUpFDg7s+Y2Y0A7l5rZnUJjkukVdpaUc3k2Wt5bkYxc4rLyEgzThrWgxvPPIhTh/dUHYG0eVH+BZebWXfAAczsaKAsoVGJtCK1dTHe/WQTz80o5rUFG6iuizG8dxd+/tURjDu0D91zs5MdokiziZIUrgcmAweY2ftAD+D8hEYl0gqsLq3gqemreLaomI3bq+jWKYtvHj2A84/ox8g+eckOTyQhmkwK7j7TzE4EhgEGLHL3moRHJpIEdTHnzYUbefzDlby1uAQDTj5oPy4o7M/YYfuRlaEKY2nfojx9dBXwuLvPD+e7mtl4d/9jwqMTaSEbtlXy9PTVPDVtFWvLKtmvczbXnDyUi47sT598jT0gqSPK7aMJ7n7vrhl332JmEwAlBWnT3J0ZK7fw8PvLeXX+BupizglDC/j510ZwyvCeZOoxUklBUZJCupmZu++qaE4H1DuXtFnVtTFembeOP723nDnFZXTJyeCK4wdz8ZgBDCrolOzwRJIqSlKYCjxtZg+E898Ny0TalC3l1TwxbRWPfbCCDduq2L9HJ247ZxTnHd5XA9SLhKL8T/hvgkTw/XD+NWBSwiISaWbLN5Xz0LvLeGFmMZU1MU4YWsDt5x3MiUN7kKaeSEU+I8rTRzHgvvAl0mbMW1PGfW8v5ZW568hIT+Prh/Xl8uMGM6xX52SHJtJqRXn66DjgFmBguL4B7u77JzY0kb3n7ny4vJQ/vrWUdxaXkJudwZVfOoBvHz+I/Tqra2qRpkS5ffQn4EfADEDdW0ir5O68uWgjf3hjCTNXbaUgN4uffnkYlxw9kLwOmckOT6TNiJIUytz9lX3ZuJmdAdwDpAOT3P32BssHAH8G8sN1bnD3KfuyL0lN7s5bi0v43WuLmV1cRr+uHbht3EguKOyvcYxF9kGUpPCmmd0JvABU7Sp095mNfSh8dPVe4DSgGJhuZpPdfUHcav8DPOPu94Vdck8BBu3dIUgqcnfe/WQTd/9rMR+t2kq/rh349XkHc+7hfdW+QOQLiJIUjgrfC+PKHDi5ic+NAZa4+zIAM3sKGAfEJwUHuoTTecDaCPFICnN3Pli6md++tpiilVvok5fDL88dzflH9FMXFCLNIMrTR2P3cdt9gdVx88V8mmB2uQX4p5ldA3QCTt3HfUkKmLlqC3e8spAPl5fSq0sOt40byTeO7K9xC0SaUaQWO2Z2FjASqH98w90nNsP+xwOPuvtdZnYM8BczGxU+Bhu//yuBKwEGDBjQDLuVtmRZyQ7ufHURr8xbT0FuNrd8bQQXjRmgOgORBIjySOr9QEdgLEGjtfOBaRG2vQboHzffLyyLdwVwBoC7f2BmOUABsDF+JXd/EHgQoLCw0CPsW9qBku1V3PP6Yp6ctprsjDSuO3UoE07YXwPZiCRQlP9dx7r7wWY2x91vNbO7gChPI00HhprZYIJkcBFwcYN1VgGnAI+a2XCCK5GS6OFLe1ReVctD7y7jwXeWUV0b4+IxA/jhKUPp0VmD2YgkWpSksDN8rzCzPsBmoHdTHwqH7bwaeJXgcdOH3X2+mU0Eitx9MvBj4CEz+xFBpfNluzrek9QTizkvfLSGO6YupGR7FV8Z1YuffnkY+/fITXZoIikjSlJ4yczygTuBmQRf3pH6PgrbHExpUPbzuOkFwHGRo5V266NVW7jlxQXMXr2VQ/rnc/8lR3DEwK7JDksk5UR5+ui2cPJ5M3sJyHF3jdEszWLjtkrumLqI52cW06NzNnddcAjnHtZXHdWJJMkek4KZnezub5jZ13ezDHd/IbGhSXtWXRvjT+8t5w9vfEJNnfP9kw7gqrFDyFUlskhSNfY/8ETgDeBru1nmBC2cRfbah8s2c9Pf57Fk4w5OG9GTm84crsFtRFqJPSYFd7/ZzNKAV9z9mRaMSdqpLeXV/HLKxzw7o5h+XTvwyGVHMvag/ZIdlojEafRa3d1jZvYzQElB9pm78/zMNfxyysds21nD9048gGtPGUqHLDU+E2ltotzA/ZeZ/QR4GijfVejupQmLStqNpSU7uOlvc/nPslKOGNiV/3fuKA7q1aXpD4pIUkRJCheG71fFlTmgQXZkj2rrYjz07nLu/tdicjLS+NXXR3NhYX89VSTSykV5JHVwSwQi7ccnG7bzk2dnM7u4jDNG9uK2c0apNbJIGxG1Q7xRwAg+2yHeY4kKStqm2roYD7yzjHv+9Qm5ORn84eLDOGt0b8x0dSDSVkTpEO9m4CSCpDAF+ArwHqCkIPUWrd/OT5+bzZziMs4c3YuJ40ZRkKurA5G2JsqVwvnAIcBH7n65mfUE/prYsKStiMWcSe8t4zevLiY3J4N7Lz6csw5usmssEWmlInWIFz6aWmtmXQi6te7f1Iek/VtfVsmPn53F+0s28+WRPfnluaPprqsDkTYtSlIoCjvEewiYAewAPkhoVNLqTZ23jhtemEtVTYw7zhvNNwr7q+5ApB2I8vTRD8LJ+81sKtDF3eckNixprcqrapn44gKeLlrNwf3y+N2Fh6pra5F2JEpF82TgKeAf7r4i4RFJqzV/bRlXP/ERKzaXc9XYA7ju1APJTE9Ldlgi0oyi/I++CzgeWGBmz5nZ+eGwmZIi3J0np63i3D/+m53VdTw54Wh++uWDlBBE2qEot4/eBt42s3TgZGAC8DCgvgpSQHlVLf/z93n87aM1nDC0gN9deKgqk0XasaiN1zoQdKF9IXA48OdEBiWtwycbtvP9x2eyrGQHPz7tQK4aO0TdVIi0c1HqFJ4BxgBTgT8Ab7t7LNGBSXL9Y9Yabnh+Lp2yM/jrFUdx7JCCZIckIi0gypXCn4Dx7l6X6GAk+WrrYtz+ykImvbecMYO78Yfxh7FfF1UhiaSKKHUKr7ZEIJJ8W8qruebJj3hvySYuO3YQN501XJXJIilGA+IKAB+v28aVfyliQ1kVvz7/YL5RqEbrIqlISUF4ec46fvLsbLp0yODp7x7NYQO6JjskEUmSPSYFMzu8sQ+6+8zmD0dakrvzv28s4bevLeaIgV2575LD2a+z6g9EUlljVwp3he85QCEwGzDgYKAIOCaxoUkiVdfGuOGFObwwcw1fP6wvvzpvNNkZGjNZJNXtMSm4+1gAM3sBONzd54bzo4BbWiQ6SYitFdV8768z+M+yUq4/7UCuOXmIOrMTESBancKwXQkBwN3nmdnwBMYkCbRyczmXPzqd4tKd3HPRoYw7tG+yQxKRViRKUphjZpP4dGCdbwLqJbUNmrGylAmPzcDdeXzCURw5qFuyQxKRViZKUrgc+D5wbTj/DnBfwiKShPjXgg1c9cRM+uR34OHLjmRwQadkhyQirVCUxmuVZnY/MMXdF7VATNLMni1azQ0vzGVU3zweuexIunXKSnZIItJKNdlc1czOBmYR9H2EmR0ajrEgbcADby/lp8/N4dgDuvPEd45SQhCRRkXpw+Bmgg7xtgK4+yxgcCKDki/O3fnllI/51SsLOevg3kz6ViGdstVWUUQaFyUp1Lh7WYMyj7JxMzvDzBaZ2RIzu2EP63zDzBaY2XwzeyLKdqVxtXUxfvrcHB58ZxmXHD2A3190mNogiEgkUX46zjezi4F0MxsK/BD4d1MfCgfluRc4DSgGppvZZHdfELfOUOBG4Dh332Jm++3LQcinqmtjXPvUR7wybz3XnjKU604dqjYIIhJZlCuFa4CRQBXwJLANuC7C58YAS9x9mbtXE4zzPK7BOhOAe919C4C7b4wauHxeVW0dP3h8Bq/MW8//nDWcH512oBKCiOyVKE8fVQA3ha+90RdYHTdfDBzVYJ0DAczsfSAduMXdpzbckJldCVwJMGDAgL0MIzVU1tTx3b/M4O3FJUwcN5JLjxmU7JBEpA2KMvLagcBPgEHx67v7yc20/6HASUA/4B0zG+3uW+NXcvcHgQcBCgsLI9VnpJKK6lomPFbEv5du5vavj+aiMUqcIrJvotQpPAvcD0wC9mb0tTVAfKf8/cKyeMXAh+5eAyw3s8UESWL6XuwnpZVX1XL5o9MpWlHKb84/hPOO6JfskESkDYuSFGrdfV9aME8HhprZYIJkcBFwcYN1/g6MBx4xswKC20nL9mFfKamypo4JjxVRtKKUuy9UP0Yi8sVFqWh+0cx+YGa9zazbrldTH3L3WuBq4FXgY+AZd59vZhPDBnGEyzab2QLgTeCn7r55H48lpVTV1vG9v87gg2WbuesbhyghiEizMPfGb9Gb2fLdFLu775+YkBpXWFjoRUVFydh1q1FbF+PqJz5i6vz1/OrroxmvOgQRaYKZzXD3wqbWi/L0kVovtyJ1MefHz85m6vz13Py1EUoIItKsGhuO82R3f8PMvr675e7+QuLCkt1xd27621z+MWstPztjGJcfp3wtIs2rsSuFE4E3gK/tZpkDSgot7NevLuKp6au55uQh/OCkIckOR0TaocaG47w5fL+85cKRPXnk/eXc99ZSvnnUAK4/7cBkhyMi7VSkbjPN7CyCri5ydpW5+8REBSWf9dKctUx8aQGnj+jJxHGj1HWFiCRMlPEU7gcuJOgDyYALgIEJjktC/166ieufnk3hwK78fvxhpKcpIYhI4kRpp3Csu18KbHH3W4FjCPssksRasHYb331sBoMKOjLp0iPJyVT31yKSWFGSws7wvcLM+gA1QO/EhSQAG7ZV8u1Hp9MpO4NHLx9DXsfMZIckIikgSp3CS2aWD9wJzCR48mhSQqNKcTur6/jOn4vYXlnDs987lj75HZIdkoikiCiN124LJ583s5eAnN2MxCbNJBZzrn9mFvPWljHp0kJG9OmS7JBEJIU01nhtt43WwmVqvJYgd722qH6QnFOG90x2OCKSYhq7Uthdo7Vd1HgtAZ6fUcy9by5l/JgBXHG8WiuLSMtrrPGaGq21oFmrt3LjC3M59oDuTBw3Um0RRCQporRT6G5mvzezmWY2w8zuMbPuLRFcqti0o4rv/3UGPfOy+eM3DyczPcpDYSIizS/Kt89TQAlwHnB+OP10IoNKJbV1Ma56fCal5dXcf8kR5HfMSnZIIpLCojyS2jvuCSSAX5jZhYkKKNXc/spCPlxeyt0XHsLIPnnJDkdEUlyUK4V/mtlFZpYWvr5BMGKafEGTZ69l0nvLuezYQZx7mMZWFpHki5IUJgBPAFXh6yngu2a23cy2JTK49mxpyQ5ueH4OhQO78n/OHJ7scEREgGiN1zq3RCCppLKmjquf+IiczHT+cPHhZGWoYllEWocoTx9d0WA+3cxuTlxI7d8vp3zMx+u28ZsLDqZXXk7THxARaSFRfqKeYmZTzKy3mY0C/gPo6mEfTZ23nsc+WMl3jh/MyQepxbKItC5Rbh9dHD5tNBcoBy529/cTHlk7VLylgp89N5uD++XxszMOSnY4IiKfE+X20VDgWuB5YCXwX2bWMdGBtTd1Mee6p2YRc/jf8YepHkFEWqUo30wvAj939+8CJwKfANMTGlU79MA7SylauYVfnDOKgd07JTscEZHditJ4bYy7bwNwdwfuMrMXExtW+7Jg7Tbufm0xZ43uzbhD+yQ7HBGRPYpypVBrZv/XzB6C+ttJGo4zoqraOq5/Zhb5HbO47ZxR6uhORFq1KEnhEYJGa8eE82uAXyQsonbm7tc+YeH67dxx3mi6dVK/RiLSukVJCge4+68JxmbG3SsA/dyNYPqKUh54Zynjx/TX46ci0iZESQrVZtaBYGAdzOwAgisHacTO6jp+8uxs+nXtwE1njUh2OCIikUSpaL4ZmAr0N7PHgeOAyxIZVHvwu9cXs3JzBU9MOIrc7Ch/ZhGR5IvSeO01M5sJHE1w2+had9+U8MjasHlrypj07nIuLOzPsQcUJDscEZHIIrWgcvfN7v6yu7+0NwnBzM4ws0VmtsTMbmhkvfPMzM2sMOq2W6vauhj//fwcunXKUu+nItLmJKxZrZmlA/cCXwFGAOPN7HM3182sM0GL6Q8TFUtLmvTecuav3cbEs0eS1zEz2eGIiOyVRPa1MAZY4u7L3L2aYByGcbtZ7zbgDqAygbG0iFWbK7j7tcWcPqInZ4zqlexwRET2WqSkYGbHm9nl4XQPMxsc4WN9gdVx88VhWfx2Dwf6u/vLEeNt1W59cT7pacat40aqkZqItElROsS7Gfhv4MawKBP46xfdsZmlAb8Ffhxh3SvNrMjMikpKSr7orhPi9Y838PrCjVx7ylB653VIdjgiIvskypXCucDZBN1m4+5riTaewhqgf9x8v7Bsl87AKOAtM1tB8HTT5N1VNrv7g+5e6O6FPXr0iLDrllVZU8etLy7ggB6duPy4KBdRIiKtU6TGa2FHeLsar0Xt4nM6MNTMBptZFnARMHnXQncvc/cCdx/k7oMIBu85292L9uoIWoEH3l7GqtIKJo4bpS6xRaRNi/IN9oyZPQDkm9kE4F/AQ019yN1rgauBV4GPgWfcfb6ZTTSzs79I0K3J6tIK/vjWEs4a3ZvjhqhNgoi0bVEar/3GzE4DtgHDCMZWeC3Kxt19CjClQdnP97DuSVG22dr86pWPSTPjprPUJkFE2r4mk4KZXQ88HTURpJIZK0uZMnc91506lD75qlwWkbYvyu2jzsA/zexdM7vazNTdJ+Du/OLlj9mvczZXfmn/ZIcjItIsmkwK7n6ru48ErgJ6A2+b2b8SHlkr9/LcdXy0ais/OX0YHbPU4Z2ItA9786jMRmA9sBnYLzHhtA1VtXXcMXUhB/XqzHlH9Et2OCIizSZK47UfmNlbwOtAd2CCux+c6MBas798sJLVpTu56azhpKep5bKItB9R7nv0B65z91mJDqYtKK+q5Y9vLeX4IQWcMLT1NaQTEfki9pgUzKyLu28D7gznu8Uvd/fSBMfWKv35gxWUlldz/ekHJjsUEZFm19iVwhPAV4EZBK2Z4++TOJByj9xsr6zhwXeWMXZYDw4f0DXZ4YiINLs9JgV3/2r4rs58Qo+8v4KtFTX86DRdJYhI+xSlovn1KGXtXVlFDQ+9u4zTRvTk4H75yQ5HRCQhGqtTyAE6AgVm1pVPbx91ocG4CKngT+8vZ3tlLdedOjTZoYiIJExjdQrfBa4D+hDUK+xKCtuAPyQ4rlalvKqWP/97BaeN6MnIPnnJDkdEJGEaq1O4B7jHzK5x9/9twZhanWeKVlO2s4bvnXhAskMREUmoKL2k/q+ZjQJGADlx5Y8lMrDWoqYuxqR3l3PkoK4cMVBPHIlI+xall9SbgZMIksIU4CvAe0BKJIUpc9exZutObjl7ZLJDERFJuCh9H50PnAKsd/fLgUOAlLix7u488PYyDujRiVMOSununkQkRURJCjvdPQbUmlkXgo7x+jfxmXZh2vJSFqzbxoQT9idNfRyJSAqI0vdRkZnlEwzBOQPYAXyQ0Khaicf+s5K8DpmMOzTlnsAVkRQVpaL5B+Hk/WY2Feji7nMSG1bybdxWyavz1nPZsYPokJWe7HBERFpEY43XDm9smbvPTExIrcMT01ZRG3MuOXpgskMREWkxjV0p3NXIMgdObuZYWo2auhhPTlvFlw7swaCCTskOR0SkxTTWeG1sSwbSmry5cCMbtlXxi3N0lSAiqSVKO4VLd1fenhuvPTujmILcbMYO0yA6IpJaojx9dGTcdA5Bm4WZtNPGa5t2VPHmwo18+/jBZKTvzRDWIiJtX5Snj66Jnw8fT30qYREl2d8/WkNtzLngiH7JDkVEpMXty0/hcqBdDrzj7jxbVMwh/fMZ2rNzssMREWlxUeoUXiR42giCJDICeCaRQSXLvDXbWLRhO784Z1SyQxERSYoodQq/iZuuBVa6e3GC4kmq52cWk5WRxtcO7pPsUEREkiJKncLbAGG/RxnhdDd3L01wbC2qLua8PHcdJw/bj7yOmckOR0QkKaLcProSmAhUAjGCEdgc2D+xobWs6StKKdlexVcP6Z3sUEREkibK7aOfAqPcfVOig0mml+esIyczjZPVRbaIpLAoTx8tBSoSHUgy1cWcV+at55SDetIxK0qeFBFpn6J8A94I/NvMPgSqdhW6+w+b+qCZnQHcA6QDk9z99gbLrwe+Q1CBXQJ8291XRg+/eUxbXsqmHVWcdbBuHYlIaouSFB4A3gDmEtQpRGJm6cC9wGlAMTDdzCa7+4K41T4CCt29wsy+D/wauDDqPprLq/PXk52Rxknq1kJEUlyUpJDp7tfvw7bHAEvcfRmAmT0FjAPqk4K7vxm3/n+AS/ZhP1+Iu/PGwo0cN6RAt45EJOVFqVN4xcyuNLPeZtZt1yvC5/oCq+Pmi8OyPbkCeGV3C8L9F5lZUUlJSYRdR7e0pJxVpRWMVQWziEikK4Xx4fuNcWXN+kiqmV0CFAIn7m65uz8IPAhQWFjou1tnX725cCOAnjoSESFa47Uomod7AAAMvUlEQVR97edoDdA/br5fWPYZZnYqcBNwortXNVyeaG8s3Miwnp3pm9+hpXctItLqJHI8henAUDMbTJAMLgIubrDtwwgqss9w942RIm5G5VW1TF9RyhUntMv+/URE9lrCxlNw91ozuxp4leCR1Ifdfb6ZTQSK3H0ycCeQCzxrZgCr3P3svT+MfTN9RSm1Mef4IQUttUsRkVYtoeMpuPsUYEqDsp/HTZ8aLczE+GDZZjLTjcKBUerNRUTav5QeT+E/SzdzaP98OmSlJzsUEZFWIWXHU9hWWcPcNWVcffLQZIciItJqpOx4CtOWlRJzOGb/7skORUSk1dhjUjCzIUDPXeMpxJUfZ2bZ7r404dEl0PQVpWSlp3HYgPxkhyIi0mo0VqfwO2Dbbsq3hcvatFmrtzK8TxdyMlWfICKyS2NJoae7z21YGJYNSlhELaAu5sxdU8ah/fKSHYqISKvSWFJo7L5Km27+u2TjDiqq6zikv24diYjEaywpFJnZhIaFZvYdYEbiQkq82au3AnBwPyUFEZF4jT19dB3wNzP7Jp8mgUIgCzg30YEl0uzirXTOzmD/gk7JDkVEpFXZY1Jw9w3AsWY2FhgVFr/s7m+0SGQJNG/tNkb27UJamiU7FBGRViVKNxdvAm82tV5bEYs5n2zYzjcK+ze9sohIitmXbi7atOItO6mormNYr87JDkVEpNVJuaSwcH3Q9EJJQUTk81IuKSzesB2AA3sqKYiINJRySWHh+u3069qB3Owo3T6JiKSWlEsKizdsZ5iuEkREdiulkkJdzFm+qZwhPXOTHYqISKuUUklh/bZKauqcgd3UaE1EZHdSKims2lwBwIBuHZMciYhI65RSSWF1qZKCiEhjUioprCqtID3N6J2fk+xQRERapZRLCn3yc8hMT6nDFhGJLKW+HVeVVujWkYhII1IqKawr20mfvDY9PpCISEKlTFJwdzbvqKagc3ayQxERabVSJimU7ayhNuYU5CopiIjsScokhU07qgAoyM1KciQiIq1XCiWFagBdKYiINCKFksKuKwUlBRGRPUmdpLBdt49ERJqSMkmhT34HTh/Rk64dlRRERPYkoUnBzM4ws0VmtsTMbtjN8mwzezpc/qGZDUpULKeP7MWDlxaSlmaJ2oWISJuXsKRgZunAvcBXgBHAeDMb0WC1K4At7j4EuBu4I1HxiIhI0xJ5pTAGWOLuy9y9GngKGNdgnXHAn8Pp54BTzEw/5UVEkiSRSaEvsDpuvjgs2+067l4LlAHdExiTiIg0ok1UNJvZlWZWZGZFJSUlyQ5HRKTdSmRSWAP0j5vvF5btdh0zywDygM0NN+TuD7p7obsX9ujRI0HhiohIIpPCdGComQ02syzgImByg3UmA98Kp88H3nB3T2BMIiLSiIxEbdjda83sauBVIB142N3nm9lEoMjdJwN/Av5iZkuAUoLEISIiSZKwpADg7lOAKQ3Kfh43XQlckMgYREQkOmtrd2vMrARYuY8fLwA2NWM4bYGOOTXomFPDFznmge7eZKVsm0sKX4SZFbl7YbLjaEk65tSgY04NLXHMbeKRVBERaRlKCiIiUi/VksKDyQ4gCXTMqUHHnBoSfswpVacgIiKNS7UrBRERaUTKJIWmxnZoq8ysv5m9aWYLzGy+mV0blnczs9fM7JPwvWtYbmb2+/DvMMfMDk/uEewbM0s3s4/M7KVwfnA4JseScIyOrLC8xcbsSCQzyzez58xsoZl9bGbHpMA5/lH4b3qemT1pZjnt8Tyb2cNmttHM5sWV7fW5NbNvhet/Ymbf2t2+okiJpBBxbIe2qhb4sbuPAI4GrgqP7QbgdXcfCrwezkPwNxgavq4E7mv5kJvFtcDHcfN3AHeHY3NsIRirA9rPmB33AFPd/SDgEIJjb7fn2Mz6Aj8ECt19FEGvCBfRPs/zo8AZDcr26tyaWTfgZuAogmELbt6VSPaau7f7F3AM8Grc/I3AjcmOK0HH+g/gNGAR0Dss6w0sCqcfAMbHrV+/Xlt5EXSu+DpwMvASYAQNejIanm+CblaOCaczwvUs2cewl8ebByxvGHc7P8e7utXvFp63l4Avt9fzDAwC5u3ruQXGAw/ElX9mvb15pcSVAtHGdmjzwkvmw4APgZ7uvi5ctB7oGU63h7/F74CfAbFwvjuw1YMxOeCzx9QexuwYDJQAj4S3zCaZWSfa8Tl29zXAb4BVwDqC8zaD9n2e4+3tuW22c54qSaHdM7Nc4HngOnffFr/Mg58O7eIxMzP7KrDR3WckO5YWlAEcDtzn7ocB5Xx6OwFoX+cYILz1MY4gIfYBOvH5WywpoaXPbaokhShjO7RZZpZJkBAed/cXwuINZtY7XN4b2BiWt/W/xXHA2Wa2gmCI15MJ7rfnh2NywGePKdKYHa1cMVDs7h+G888RJIn2eo4BTgWWu3uJu9cALxCc+/Z8nuPt7blttnOeKkkhytgObZKZGUEX5B+7+2/jFsWPVfEtgrqGXeWXhk8xHA2UxV2mtnrufqO793P3QQTn8Q13/ybwJsGYHPD5423TY3a4+3pgtZkNC4tOARbQTs9xaBVwtJl1DP+N7zrmdnueG9jbc/sqcLqZdQ2vsk4Py/ZesitYWrAi50xgMbAUuCnZ8TTjcR1PcGk5B5gVvs4kuJ/6OvAJ8C+gW7i+ETyJtRSYS/B0R9KPYx+P/STgpXB6f2AasAR4FsgOy3PC+SXh8v2THfc+HuuhQFF4nv8OdG3v5xi4FVgIzAP+AmS3x/MMPElQb1JDcFV4xb6cW+Db4fEvAS7f13jUollEROqlyu0jERGJQElBRETqKSmIiEg9JQUREamnpCAiIvWUFKRFmJmb2V1x8z8xs1uaaduPmtn5Ta/5hfdzQdhD6ZuJ3leymdn/SXYMkhxKCtJSqoCvm1lBsgOJF9c6NoorgAnuPjZR8bQiSgopSklBWkotwVCCP2q4oOEvfTPbEb6fZGZvm9k/zGyZmd1uZt80s2lmNtfMDojbzKlmVmRmi8P+kXaNuXCnmU0P+57/btx23zWzyQStZBvGMz7c/jwzuyMs+zlBQ8E/mdmdu/nMf4efmW1mt4dlh5rZf8J9/y2uT/y3zOzuMN6PzexIM3sh7Af/F+E6gywYO+HxcJ3nzKxjuOyUsGO8uRb0xZ8dlq8ws1vNbGa47KCwvFO43rTwc+PC8svC/U4N9/3rsPx2oIOZzQr338nMXg6PbZ6ZXbgX513ammS35tMrNV7ADqALsIKgX5qfALeEyx4Fzo9fN3w/CdhK0DVwNkFfLreGy64Ffhf3+akEP3KGErQKzSHob/5/wnWyCVoEDw63Ww4M3k2cfQi6WOhB0BHdG8A54bK32E3rYII+7v8NdAznd7U+nQOcGE5PjIv3LeCOuONYG3eMxQStWQcRtFQ/Llzv4fBvlkPQG+aBYfljBJ0gEv5trwmnfwBMCqd/CVwSTucTtOzvBFwGLAvPRw6wEugffw7C6fOAh+Lm85L970mvxL10pSAtxoPeWx8jGDwlqunuvs7dqwia9v8zLJ9L8MW5yzPuHnP3Twi+6A4i6P/lUjObRdCdeHeCpAEwzd2X72Z/RwJvedARWy3wOPClJmI8FXjE3SvC4yw1szwg393fDtf5c4Pt7Op7ay4wP+4Yl/Fpx2ar3f39cPqvBFcqwwg6ilu8h+3u6hBxBp/+fU4Hbgj/Dm8RJIAB4bLX3b3M3SsJrpoG7ub45gKnmdkdZnaCu5c18feQNmxv7qeKNIffATOBR+LKaglvZZpZGpAVt6wqbjoWNx/js/9+G/bX4gT9xFzj7p/pGMzMTiK4Ukim+ONoeIy7jmt3xxR1u3Vx2zHgPHdfFL+imR3VYN/xn/l0p+6LLRj28UzgF2b2urtPjBCLtEG6UpAW5e6lwDN8OowiBLc9jginzwYy92HTF5hZWljPsD/BiFSvAt+3oGtxzOxACwanacw04EQzK7BgGNfxwNtNfOY14PK4e/7dwl/TW8zshHCd/4qwnYYGmNkx4fTFwHvhcQ0ysyF7sd1XgWvC3kYxs8Mi7Lsm7u/WB6hw978CdxJ02y3tlK4UJBnuAq6Om38I+IeZzSaoG9iXX/GrCL7QuwDfc/dKM5tEcAtlZviFWAKc09hG3H2dmd1A0EWzAS+7+z+a+MxUMzsUKDKzamAKwdM73wLuD5PFMuDyvTymRQRjbj9McGvnvvC4LgeeDZ+cmg7c38R2biO4QpsTXoktB77axGceDNefSXDL704zixH05Pn9vTwOaUPUS6pIK2TB0KoveTBovUiL0e0jERGppysFERGppysFERGpp6QgIiL1lBRERKSekoKIiNRTUhARkXpKCiIiUu//A6V169rlbGMAAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pca = PCA()\n",
    "pca.fit(X_train)\n",
    "\n",
    "\n",
    "plt.plot(np.cumsum(pca.explained_variance_ratio_))\n",
    "plt.xlabel('Number of components')\n",
    "plt.ylabel('Cumulative explained variance')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Seems to have much lower recall with PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pca = PCA(n_components=2000)\n",
    "#pca.fit(X_train)\n",
    "#X_train = pca.transform(X_train)\n",
    "\n",
    "#X_test = pca.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Truncated SVD Dimensonality reduction (Does nothing for now)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Cumulative explained variance')"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd8FWX2x/HPIXSkSK+h96JiALF3sYO4a1v7imvZ5k8FO6K7imWbuva67q4FUFFRLKCuDSlKQif0ANJ7IKSc3x8zrFc2JIPJzb1Jvu/XK697Z+aZyXnSTmaemfOYuyMiIvJTVUl0ACIiUr4pkYiISIkokYiISIkokYiISIkokYiISIkokYiISIkokYiISIkokYiISIkokYiISIlUTXQAZaFx48berl27RIchIlKuTJ8+fb27NymuXaVIJO3atWPatGmJDkNEpFwxs2VR2unSloiIlIgSiYiIlIgSiYiIlIgSiYiIlIgSiYiIlIgSiYiIlIgSiYiIlIgSiYhIBTT/+2088P48ymI69UrxQKKISGWxO6+Av3+SyWOTM6lbsxq/OKwtLRvUiuvnjOsZiZkNMrP5ZpZpZiMK2V7DzF4Nt08xs3bh+v5m9l34MdPMhkQ9pohIZTVzxWbOfORz/vLRQk7r3YIPf3903JMIxPGMxMxSgMeAk4AsYKqZjXf3OTHNrgQ2uXsnMzsfGA2cB8wC0tw9z8xaADPN7G3AIxxTRKRS2bk7nz99OJ9nP19C07o1efbSNE7o3qzMPn88L231BzLdfTGAmb0CnA3E/tE/GxgZvh8DPGpm5u7ZMW1qEiSQqMcUEak0vly0nhFjM1i+MZsLB6Qy4tRu1KtZrUxjiGciaQWsiFnOAgbsq0149rEFaASsN7MBwHNAW+DicHuUY4qIVHhbd+Vy34R5/Pub5bRtVJt/X3UYAzs2SkgsSTvY7u5TgJ5m1h140cze25/9zWwYMAwgNTU1DhGKiCTGR3PWcNubGazblsOwozvw+xO7UKt6SsLiiWciWQm0iVluHa4rrE2WmVUF6gMbYhu4+1wz2w70injMPfs9BTwFkJaWFv/730RE4mzD9hzufnsO42euolvzujx1cRoHtWmQ6LDimkimAp3NrD3BH/vzgQv3ajMeuBT4CjgXmOTuHu6zIryc1RboBiwFNkc4pohIheLujJ+5ipHjZ7M9J48bTurCr47pSPWqyfEoYNwSSZgErgcmAinAc+4+28xGAdPcfTzwLPAPM8sENhIkBoAjgRFmlgsUANe6+3qAwo4Zrz6IiCTaqs07uf3NWUyat5aD2zTggXP70KVZ3USH9SNWFk89JlpaWpprhkQRKU8KCpx/T13OfRPmkV/g3HhKVy47vB0pVazMYjCz6e6eVly7pB1sFxGprJas38GIselMWbKRIzo14r4hfUhtVDvRYe2TEomISJLIyy/guS+W8PAHC6hetQqjh/bm52ltMCu7s5CfQolERCQJzF29leFj00nP2sJJPZpx7+BeNKtXM9FhRaJEIiKSQDl5+Tw2KZO/f7KIBrWr8diFfTmtd/OkPwuJpUQiIpIgM5ZvYviYdBau3c45h7TijjN6cGCd6okOa78pkYiIlLHs3Xk8NHEBz3+5hBb1avL85f04rmvTRIf1kymRiIiUoS8y1zNiXDorNu7k4sPacvOgrtQt4yKLpU2JRESkDGzZmcsf353Lq9NW0L5xHV4ddhgDOiSmyGJpUyIREYmzD2Z/z+1vzmLDjt386piO/O7EztSslrgii6VNiUREJE7Wbcth5NuzeTd9Nd1b1OPZS/vRu3X9RIdV6pRIRERKmbvzxrcrGfXOHLJz8rnplK4MO7oD1VKSo8hiaVMiEREpRSs37+S2NzL4ZP46+qYGRRY7NU2uIoulTYlERKQUFBQ4/5yyjPvfm4cDI8/swcUDy7bIYqIUm0jMrBnwR6Clu59qZj2Age7+bNyjExEpBxav286IsRl8s3QjR3VuzB+H9KZNw+QtsljaopyRvAA8D9wWLi8AXiWYS0REpNLKyy/g6f8s4c8fLaBm1So8eG4fzj20dbkqb1IaoiSSxu7+mpndAv+dsCo/znGJiCS12au2MHxsOrNWbmVQz+aMGtyTpnXLR5HF0hYlkewws0aAA5jZYcCWuEYlIpKkduXm88ikhTzx6WIOrF2dxy/qy6m9WyQ6rISKkkhuIJhbvaOZfQE0IZhfXUSkUpm+bCM3j0ln0bodDO3bmjvO6E6D2uWvyGJpKzaRuPsMMzsG6AoYMN/dc+MemYhIktiRk8eDE+fz4ldLaVm/Fi9e0Z9jujRJdFhJI8pdW9cB/3T32eHygWZ2gbv/Pe7RiYgk2GcL1nHLuAxWbdnJpQPbceMpXTmghp6ciBXlq3GVuz+2Z8HdN5nZVYASiYhUWJuzd3Pvu3MZMz2LDk3q8PrVA0lr1zDRYSWlKIkkxczM3fcMtqcAuigoIhXWexmrueOt2WzK3s11x3Xk18dXrCKLpS1KInkfeNXMngyXrw7XiYhUKGu37eKut2bz3qzv6dmyHi9e0Y+eLStekcXSFiWRDCdIHteEyx8Cz8QtIhGRMubujJmexb3vzmVnbj43D+rKVUdV3CKLpS3KXVsFwOPhh4hIhbJiYza3vpHBfxaup1+7A7l/aB86Njkg0WGVK1Hu2joCGAm0Ddsb4O7eIb6hiYjET0GB89JXS3lg4nwMuOfsnlw0oC1VKkGRxdIW5dLWs8DvgemASqOISLmXuXYbw8dmMH3ZJo7p0oQ/DOlF6wMrT5HF0hYlkWxx9/fiHomISJzl5hfw1GeL+etHC6ldI4U//fwghhzSqtIVWSxtURLJZDN7EBgH5OxZ6e4z4haViEgpm7VyCzePSWfO6q2c3rsFI8/qSZO6NRIdVoUQJZEMCF/TYtY5cHzphyMiUrp25ebz148X8tRni2lYpzpP/OJQBvVqnuiwKpQod20dVxaBiIiUtqlLNzJ8TDqL1+/g52mtue20HtSvXS3RYVU4kQrGmNnpQE/gv8X23X1UvIISESmJ7Tl5PPD+PF76ahmtD6zFy1cO4MjOjRMdVoUV5fbfJ4DawHEEDyKeC3wT57hERH6SyfPXctu4DFZv3cUVR7TnxlO6ULu6iizGU5Sv7uHu3sfM0t39bjN7GNBdXCKSVDbt2M0978xh3Lcr6dT0AMb86nAObXtgosOqFKIkkp3ha7aZtQQ2AJV7OjARSRruzoSM77lr/Cw2Z+fym+M7cd3xnahRVUUWy0qURPKOmTUAHgRmENyxpVpbIpJwa7fu4vY3Z/HBnDX0blWfl64YQI+W9RIdVqUT5a6te8K3Y83sHaCmu2vOdhFJGHfn9WlZ3PPuHHbnFXDLqd248sj2VFWRxYTYZyIxs+PdfZKZnVPINtx9XHxDExH5X8s3ZHPLG+l8kbmB/u0bMnpoH9o3rpPosCq1os5IjgEmAWcWss0JnnQXESkT+QXOC18u5aGJ80mpYtw7uBcX9k9VkcUksM9E4u53mVkV4D13f+2nHNzMBgF/BVKAZ9z9/r221wBeAg4lGMQ/z92XmtlJwP0EMzHuBm5y90nhPp8QDPbvuQngZHdf+1PiE5HyYeGabdw8Np1vl2/muK5N+MOQ3rRsUCvRYUmoyDESdy8ws5uB/U4k4ZS8jwEnAVnAVDMb7+5zYppdCWxy905mdj4wGjgPWA+c6e6rzKwXMBFoFbPfRe4+bX9jEpHyZXdeAU98uohHJ2VSp0YKfznvYM4+uKWKLCaZKHdtfWRmNwKvAjv2rHT3jcXs1x/IdPfFAGb2CnA2EJtIziaY6wRgDPBoOD/8tzFtZgO1zKyGu+cgIpVCetZmbh6Tzrzvt3HmQS2568weND5ARRaTUZREcl74el3MOgeKm9iqFbAiZjmLHwpA/k8bd88zsy1AI4Izkj2GAjP2SiLPm1k+MBa41919709uZsOAYQCpqanFhCoiyWLn7nz+8tECnv7PYprUrcHTl6RxUo9miQ5LihDl9t/2ZRFIYcysJ8HlrpNjVl/k7ivNrC5BIrmYYJzlR9z9KeApgLS0tP9JNCKSfL5evIERY9NZuiGbC/q34ZbTulOvpoosJruoRRt7AT34cdHG//njvZeVQJuY5dbhusLaZJlZVaA+waA7ZtYaeAO4xN0XxXzeleHrNjP7F8EltOJiEZEktm1XLve/N49/TllOasPa/OuXAzi8k4oslhdRijbeBRxLkEgmAKcCn1P8H++pQGcza0+QMM4HLtyrzXjgUuArgmKQk9zdwyfp3wVGuPsXMbFUBRq4+3ozqwacAXxUXB9EJHlNmreG296YxZqtu/jlke35v5O7Uqu6ypuUJ1HOSM4FDgK+dffLzawZ8HJxO4VjHtcT3HGVAjzn7rPNbBQwzd3HE8wH/w8zywQ2EiQbgOuBTsCdZnZnuO5kgsH+iWESSSFIIk9H7KuIJJGNO3Yz6u3ZvPndKro0O4C/X3Q4h6SqyGJ5ZIWMU/+4gdk37t7fzKYTlJLfBsx1925lEWBpSEtL82nTdLewSDJwd95OX83I8bPZtiuXa4/txHXHdaJ6VZU3STZmNt3d04prF+WMZFp4qelpYDqwneBSlIjIfvl+S1Bk8aO5aziodX1GnzuAbs1VZLG8i3LX1rXh2yfM7H2gnrunxzcsEalI3J1Xpq7gj+/OJbeggNtP787lR7QnReVNKoQog+3jgVeAt9x9adwjEpEKZdmGHYwYm8FXizcwsEMj7h/am7aNVGSxIolyaethgocS7zOzqQRJ5R133xXXyESkXMsvcJ7/YgkPfTCfalWqcN85vTm/XxuVN6mAolza+hT4NKyddTxwFfAcoAubIlKo+d8HRRZnrtjMid2bcu/g3jSvX7P4HaVcivpAYi2CcvLnAX2BF+MZlIiUT7vzCnhsciZ//ySTujWr8bcLDuHMPi10FlLBRRkjeY3g6fH3gUeBT929IN6BiUj58t2Kzdw8ZiYL1mxn8MEtufPMnjSsUz3RYUkZiHJG8ixwgbvnxzsYESl/du7O5+EP5vPcF0toVq8mz12WxvHdVGSxMokyRjKxLAIRkfLny0XrGTE2g+Ubs7loQCojTu1GXRVZrHQijZGIiMTauiuX+ybM5d/frKBdo9q8MuwwDuvQKNFhSYIokYjIfvlozhpuezODddtyuProDvzuxC4qsljJ7TORmFnfonZ09xmlH46IJKv123O4++05vD1zFd2a1+XpS9Lo07pBosOSJFDUGcnD4WtNIA2YCRjQB5gGDIxvaCKSDNydt75bxd1vz2Z7Th43nNSFXx3TUUUW5b/2mUjc/TgAMxsH9HX3jHC5Fz/Msy4iFdiqzTu5/c1ZTJq3lkNSG/DA0D50blY30WFJkokyRtJ1TxIBcPdZZtY9jjGJSIIVFDj/+mY59783j/wC584zenDp4e1UZFEKFSWRpJvZM/wwmdVFgKr/ilRQS9bvYMTYdKYs2cgRnRpx35A+pDaqneiwJIlFSSSXA9cAvw2XPwMej1tEIpIQefkFPPv5Ev704QKqV63CA0P78LO01ipvIsWK8kDiLjN7Apjg7vPLICYRKWNzVm1l+Nh0MlZu4eQezbhncC+a1VORRYkmSq2ts4AHgepAezM7GBjl7mfFOzgRia+cvHwenZTJ458sokHtajx2YV9O691cZyGyX6Jc2rqLoGjjJwDu/p2ZtY9nUCISf9OXbWL42HQy127nnL6tuOP0HhyoIovyE0RJJLnuvmWv/1A8TvGISJxl787jwYnzeeHLpbSoV5PnL+/HcV2bJjosKceiJJLZZnYhkGJmnYHfAF/GNywRiYfPF65nxLh0sjbt5JKBbbl5UDcOqKFKSVIyUX6Cfg3cBuQA/wYmAvfEMygRKV1bsnP5w4Q5vDYti/aN6/Da1QPp375hosOSCiLKXVvZBInktviHIyKl7f1Z33PHW7PYuGM31xzbkd+e0Jma1VRkUUpPlLu2ugA3Au1i27v78fELS0RKat22HEaOn827Gavp0aIez1/Wj16t6ic6LKmAolzaeh14AngG0CyJIknO3Rk3YyWj3pnDzt353HRKV4Yd3YFqKSqyKPERJZHkubueZBcpB1Zu3smt4zL4dME6Dm17IKOH9qFT0wMSHZZUcFESydtmdi3wBsGAOwDuvjFuUYnIfikocF6esozR783DgZFn9uCSge2ooiKLUgaiJJJLw9ebYtY50KH0wxGR/bVo3XZGjE1n6tJNHNW5MX8c0ps2DVVkUcpOlLu29BS7SBLKzS/g6f8s5i8fLaRWtRQe+tlBDO3bSuVNpMwVNdXu8e4+yczOKWy7u4+LX1giUpRZK7cwfGw6s1dt5dRezbn77J40rasii5IYRZ2RHANMAs4sZJsDSiQiZWxXbj6PTFrIE58u5sDa1Xn8or6c2rtFosOSSq6oqXbvCl8vL7twRGRfpi3dyM1j01m8bgfnHtqa20/vToPaKrIoiRepyI6ZnQ70BP577uzuo+IVlIj8YEdOUGTxxa+W0rJ+LV66oj9Hd2mS6LBE/ivKk+1PALWB4wgeSjwX+CbOcYkI8OmCddw6LoNVW3Zy6cB23HRKV+qoyKIkmSg/kYe7ex8zS3f3u83sYeC9eAcmUpltzt7NPe/MZeyMLDo2qcPrVw8krZ2KLEpyipJIdoav2WbWEtgAaHRPJE7ey1jNHW/NZlP2bq4/rhPXH99JRRYlqUVJJO+YWQOC6XZnENyx9UxcoxKphNZu3cWdb83m/dnf07NlPV68oh89W6rIoiS/Yqu4ufs97r7Z3ccCbYFu7n5HlIOb2SAzm29mmWY2opDtNczs1XD7FDNrF64/ycymm1lG+Hp8zD6HhuszzexvpqevpJxzd16ftoIT//Qpk+avZfigbrx13RFKIlJuFPVAYqEPIobbin0g0cxSgMeAk4AsYKqZjXf3OTHNrgQ2uXsnMzsfGA2cB6wHznT3VWbWi2AyrVbhPo8DVwFTgAnAIDRmI+XUio3Z3PpGBv9ZuJ5+7Q7k/qF96NhERRalfCnq0lZhDyLuEeWBxP5AprsvBjCzV4CzgdhEcjYwMnw/BnjUzMzdv41pMxuoZWY1gIZAPXf/OjzmS8BglEiknMkvcF76aikPTpyPAfec3ZOLBrRVkUUpl4p6ILGkDyK2AlbELGcBA/bVxt3zzGwL0IjgjGSPocAMd88xs1bhcWKP2QqRciRz7TaGj81g+rJNHNOlCX88pzetGtRKdFgiP1mU50gaAXcBRxKciXwOjHL3DXGODTPrSXC56+SfsO8wYBhAampqKUcmsv9y8wt48tNF/O3jTGrXSOFPPz+IIYeoyKKUf1Hu2noF+IzgzADgIuBV4MRi9lsJtIlZbh2uK6xNlplVBeoT3F6MmbUmmAPlEndfFNO+dTHHBMDdnwKeAkhLS/NiYhWJq1krt3DTmHTmrt7K6X1aMPLMnjSpWyPRYYmUiiiJpIW73xOzfK+ZnRdhv6lAZzNrT/DH/nzgwr3ajCeY7+QrgifmJ7m7h7cbvwuMcPcv9jR299VmttXMDiMYbL8EeCRCLCIJsSs3n798tJCn/7OYhnWq8+TFh3JKz+aJDkukVEVJJB+Ed1S9Fi6fS3AXVZHCMY/rw7YpwHPuPtvMRgHT3H088CzwDzPLBDYSJBuA64FOwJ1mdme47mR3XwtcC7wA1CIYZNdAuySlKYs3MGJcBkvW7+C8tDbcelp36teuluiwREqduRd91cfMtgF1gPxwVQqwI3zv7l4vfuGVjrS0NJ82bVqiw5BKYtuuXB54fz7/+HoZbRrW4r4hfTiyc+NEhyWy38xsurunFdcuygyJdUsnJJGKb/L8tdw2LoPVW3dxxRHtufGULtSuriKLUrEV+2S7mV2513KKmd0Vv5BEyp9NO3Zzw6vfcfnzU6lToypjrzmcO8/soSQilUKUn/ITzGwowVPojYDngU/jGpVIOeHuvJuxmrvems2Wnbn85vhOXHd8J2pUVZFFqTyiXNq6MLxLK4NgbOTC2DupRCqrNVt3cfubs/hwzhp6t6rPy78cQPcWST9kKFLqojyQ2Bn4LTAW6A5cbGbfunt2vIMTSUbuzmvTVnDvu3PZnVfALad248oj21M1pdgrxSIVUpRLW28D17v7R2Gl3RsInhHpGdfIRJLQ8g3ZjBiXzpeLNjCgfUPuH9qH9o3rJDoskYSKkkj6u/tWCO71BR42s7fjG5ZIcskvcF74cikPTZxPShXjD0N6cUG/VBVZFCFaIskzszuAVHe/KrzU1QVYEN/QRJLDgjXbuHlMOt+t2Mzx3ZryhyG9aFFfRRZF9oiSSJ4HpgMDw+WVwOvAO/EKSiQZ7M4r4PFPFvHo5IUcUKMqfz3/YM46qKWKLIrsJUoi6eju55nZBQDunq1ZCaWim7liM8PHpjPv+22ceVBLRp7Zg0YHqMiiSGGiJJLdZlaLoIQ8ZtYRyIlrVCIJsnN3Pn/+aAHP/GcxTerW4OlL0jipR7NEhyWS1KIkkruA94E2ZvZP4AjgsngGJZIIXy3awC3j0lm6IZsL+qdyy2ndqFdTRRZFihPlgcQPzWwGcBhgwG/dfX0xu4mUG1t35XL/e/P415TltG1Um39dNYDDO6rIokhUkQoBhbMhvhvnWETK3KR5a7h13CzWbtvFVUe154aTulKrusqbiOwPVZSTSmnD9hxGvTOHt75bRddmdXni4kM5uE2DRIclUi4pkUil4u6Mn7mKu9+ew7ZdufzuxM5ce2wnqldVeRORnypSIjGzI4HO7v68mTUBDnD3JfENTaR0rd6yk9vfmMXH89ZyUJsGPDC0D12ba7odkZKKUrTxLiAN6ErwcGI14GWCu7dEkl5BgfPK1BXcN2EuuQUF3H56dy4/oj0pKm8iUiqinJEMAQ4BZgC4+yoz079xUi4sXb+DEePS+XrxRgZ2aMT9Q3vTtpGKLIqUpkgPJLq7m9meBxL1WyhJLy+/gOe/WMrDH86nWpUq3H9Ob87r10blTUTiIEoiec3MngQamNlVwBXA0/ENS+Snm/f9VoaPSWdm1hZO7N6Uewf3pnn9mokOS6TCivJA4kNmdhKwlWCc5E53/zDukYnsp5y8fB6bvIi/T86kfq1qPHLBIZzRp4XOQkTiLMpg+w3Aq0oeksy+Xb6J4WPTWbBmO4MPbsmdZ/akYZ3qiQ5LpFKIcmmrLvCBmW0EXgVed/c18Q1LJJrs3Xk8/MECnvtiCc3r1eS5y9I4vpuKLIqUpSiXtu4G7jazPsB5wKdmluXuJ8Y9OpEifJm5nhHjMli+MZtfHJbK8EHdqKsiiyJlbn+ebF8LfA9sAJrGJxyR4m3Zmct9E+byytQVtGtUm1eGHcZhHRolOiyRSivKGMm1wM+BJgQzI17l7nPiHZhIYT6Y/T23vzmL9dtzuPqYDvz+xC7UrKYiiyKJFOWMpA3wO3f/Lt7BiOzL+u05jBw/m3fSV9OteV2euTSNPq1VZFEkGewzkZhZPXffCjwYLjeM3e7uG+McmwjuzpvfreTut+eQnZPP/53UhauP6agiiyJJpKgzkn8BZwDTCabZjb0Z34EOcYxLhFWbd3LbGxlMnr+OQ1KDIoudm6k6j0iy2Wcicfczwtf2ZReOSFBk8Z/fLGf0e/PIL3DuPKMHlx7eTkUWRZJUlMH2j939hOLWiZSGxeu2M2JcBt8s2ciRnRpz3zm9adOwdqLDEpEiFDVGUhOoDTQ2swP54dJWPaBVGcQmlUhefgHPfL6EP3+4gOpVq/DA0D78LK21ypuIlANFnZFcDfwOaEkwTrLnN3or8Gic45JKZM6qrdw8diazVm7l5B7NuGdwL5rVU5FFkfKiqDGSvwJ/NbNfu/sjZRiTVBI5efk8OimTxz9ZRIPa1fj7RX05tVdznYWIlDNRSqQ8Yma9gB5AzZj1L8UzMKnYpi8Liixmrt3OOX1bccfpPThQRRZFyqWoU+0eS5BIJgCnAp8DSiSy33bk5PHQB/N54cultKxfixcu78exXVVxR6Q8i/Jk+7nAQcC37n65mTUjmLNdZL/8Z+E6bhmXQdamnVwysC03D+rGATX2p9ybiCSjKL/FO929wMzyzKweQfHGNnGOSyqQLdm53PvuHF6fnkWHxnV47eqB9G/fsPgdRaRciFJnYpqZNSCYXnc6MAP4KsrBzWyQmc03s0wzG1HI9hpm9mq4fYqZtQvXNzKzyWa23cwe3WufT8Jjfhd+6LpIEnt/1vec+OdPGfftSq49tiMTfnuUkohIBRNlsP3a8O0TZvY+UM/d04vbz8xSgMeAk4AsYKqZjd+rcvCVwCZ372Rm5wOjCeY82QXcAfQKP/Z2kbtPKy4GSZy123YxcvxsJmR8T48W9Xj+sn70alU/0WGJSBwU9UBi36K2ufuMYo7dH8h098XhPq8AZwOxieRsYGT4fgzwqJmZu+8APjezTsV3QZKJuzNuxkpGvTOHnbn53HRKV4Yd3YFqKSqyKFJRFXVG8nAR2xw4vphjtwJWxCxnAQP21cbd88xsC9AIWF/MsZ83s3xgLHCvu/veDcxsGDAMIDU1tZjDSWnI2pTNrW/M4rMF6zi07YGMHtqHTk0PSHRYIhJnRT2QeFxZBrIfLnL3lWZWlyCRXEwhtyK7+1PAUwBpaWn/k2ik9BQUOP/4ehmj358HwN1n9eTiw9pSRUUWRSqFKM+RXFLY+ggPJK7kx3d3tQ7XFdYmy8yqAvUJpvLdJ3dfGb5uM7N/EVxC0zMtCbJo3XaGj0ln2rJNHNW5MX8coiKLIpVNlNt/+8W8rwmcQHDnVnF/vKcCnc2sPUHCOB+4cK8244FLCe4COxeYVNhlqj3CZNPA3debWTWC+VI+itAHKWW5+QU89dli/vrxQmpVS+Ghnx3E0L6tVN5EpBKKctfWr2OXw1uBX4mwX56ZXQ9MBFKA59x9tpmNAqa5+3jgWeAfZpYJbCRINns+z1KCSsPVzWwwcDKwDJgYJpEUgiTydJSOSumZtXILw8emM3vVVk7r3ZyRZ/WkaV0VWRSprH7KY8U7gEiTXbn7BIKyKrHr7ox5vwv42T72bbePwx4aKUopdbty8/nbxwt58rPFHFi7Ok/8oi+DerVIdFgikmA3R8exAAAQaElEQVRRxkjeJrhLC4IHGHsAr8UzKEk+U5duZPiYdBav38HPDm3N7af3oH7taokOS0SSQJQzkodi3ucBy9w9K07xSJLZnpPHA+/P46WvltGqQS1euqI/R3dpkuiwRCSJRBkj+RQgrLNVNXzf0N03xjk2SbBPF6zj1nEZrNqyk8sOb8dNp3SljoosisheolzaGgaMIihbUkAwU6IDHeIbmiTK5uzdjHpnDuNmrKRjkzqM+dVADm2r+lgiUrgo/17eBPRy9+KeNpcKYELGau58axabs3O5/rhOXH98J2pWS0l0WCKSxKIkkkVAdrwDkcRau3UXd7w1i4mz19CrVT1evKI/PVuqyKKIFC9KIrkF+NLMpgA5e1a6+2/iFpWUGXfn9elZ3PvOHHblFTB8UDeuOqo9VVVkUUQiipJIngQmARkEYyRSQazYmM0t4zL4PHM9/ds15P6hvenQREUWRWT/REkk1dz9hrhHImUmv8B56aulPPD+fKoY3DO4Fxf1T1WRRRH5SaIkkvfCO7fe5seXtnT7bzmUuXYbN49JZ8byzRzbtQl/GNKbVg1qJTosESnHoiSSC8LXW2LW6fbfciY3v4AnPlnEI5MyqV0jhT+fdxCDD1aRRREpuSgPJEaqqyXJKyNrCzeNmcm877dxep8W3H1WTxofUCPRYYlIBRHP+UgkwXbl5vPnjxbw9GeLaXxADZ68+FBO6dk80WGJSAUTz/lIJIGmLN7AiHEZLFm/g/PS2nDr6d2pX0tFFkWk9MVtPhJJjG27chn9/jxe/no5bRrW4p+/HMARnRonOiwRqcDiOh+JlK3J89Zy2xsZrN66iyuPbM//ndyF2tVVZFFE4kvzkVQAG3fs5p535vDGtyvp3PQAxl5zOH1TD0x0WCJSSWg+knLM3XknfTUjx89my85cfnNCZ647riM1qqrIooiUnX0mEjPrBDTbMx9JzPojzKyGuy+Ke3SyT2u27uK2N2bx0dw19Gldn5d/OYDuLeolOiwRqYSKOiP5Cz9+CHGPreG2M+MSkRTJ3Xl16gr+MGEuu/MKuPW0blxxhIosikjiFJVImrl7xt4r3T3DzNrFLSLZp+UbshkxLp0vF21gQPuGjB7ah3aN6yQ6LBGp5IpKJA2K2KbiTGUov8B5/oslPPTBfKpWqcIfh/Tm/H5tVGRRRJJCUYlkmpld5e5Px640s18C0+Mbluwx//ttDB+bzncrNnN8t6b8YUgvWtRXHheR5FFUIvkd8IaZXcQPiSMNqA4MiXdgld3uvAL+/kkmj03OpG7Navz1/IM566CWKrIoIklnn4nE3dcAh5vZcUCvcPW77j6pTCKrxGau2MzNY9KZv2YbZx3UkrvO7EEjFVkUkSQVpUTKZGByGcRS6e3cnc+fPpzPs58voWndmjxzSRon9miW6LBERIqk+hlJ4qtFGxgxLp1lG7K5cEAqI07tRr2aKrIoIslPiSTBtu7K5b4J8/j3N8tp26g2/7pqAId3VJFFESk/lEgS6KM5a7jtzQzWbcth2NEd+P2JXahVXeVNRKR8USJJgA3bc7j77TmMn7mKrs3q8uTFaRzcpqjHdkREkpcSSRlyd8bPXMXI8bPZnpPH70/swjXHdqR6VZU3EZHyS4mkjKzespPb35jFx/PWcnCbBjxwbh+6NKub6LBEREpMiSTOCgqcf09dzn0T5pFXUMDtp3fn8iPak6LyJiJSQSiRxNHS9TsYMS6drxdv5PCOjbj/nD6kNqqd6LBEREqVEkkc5OUX8NwXS3j4gwVUT6nC/ef05rx+bVTeREQqJCWSUjZ39VaGj00nPWsLJ3Zvxr2De9G8fs1EhyUiEjdKJKUkJy+fxyYv4u+TM6lfqxqPXHAIZ/RpobMQEanwlEhKwYzlmxg+Jp2Fa7cz5JBW3HFGDxrWqZ7osEREykRcH2Aws0FmNt/MMs1sRCHba5jZq+H2KXtmXjSzRmY22cy2m9mje+1zqJllhPv8zRL4L3/27jzueWcOQx//ku05eTx/WT/+fN7BSiIiUqnE7YzEzFKAx4CTgCxgqpmNd/c5Mc2uBDa5eyczOx8YDZwH7ALuIChf3+vHR+Zx4CpgCjABGAS8F69+7MsXmesZMS6dFRt38ovDUhk+qBt1VWRRRCqheJ6R9Acy3X2xu+8GXgHO3qvN2cCL4fsxwAlmZu6+w90/J0go/2VmLYB67v61uzvwEjA4jn34H1t25jJ8TDoXPTOFqlWq8Oqww7h3cG8lERGptOI5RtIKWBGznAUM2Fcbd88zsy1AI2B9EcfM2uuYrUol2gg+mP09t785i/Xbc7j6mKDIYs1qKrIoIpVbhR1sN7NhwDCA1NTUEh1r3bYcRr49m3fTV9OteV2euTSNPq1VZFFEBOKbSFYCbWKWW4frCmuTZWZVgfrAhmKO2bqYYwLg7k8BTwGkpaX5fkX+wzF487uV3P32HLJz8rnx5C5cfUxHqqWoyKKIyB7xTCRTgc5m1p7gj/35wIV7tRkPXAp8BZwLTArHPgrl7qvNbKuZHUYw2H4J8Eg8gs/NL2DYS9OYPH8dfVODIoudmqrIoojI3uKWSMIxj+uBiUAK8Jy7zzazUcA0dx8PPAv8w8wygY0EyQYAM1sK1AOqm9lg4OTwjq9rgReAWgR3a8Xljq1qKVXo0OQAju7ShEsGtlORRRGRfbAiTgAqjLS0NJ82bVqiwxARKVfMbLq7pxXXThf7RUSkRJRIRESkRJRIRESkRJRIRESkRJRIRESkRJRIRESkRJRIRESkRJRIRESkRCrFA4lmtg5Y9hN3b8y+qxFXVOpz5VDZ+lzZ+gsl73Nbd29SXKNKkUhKwsymRXmysyJRnyuHytbnytZfKLs+69KWiIiUiBKJiIiUiBJJ8Z5KdAAJoD5XDpWtz5Wtv1BGfdYYiYiIlIjOSEREpESUSEJmNsjM5ptZppmNKGR7DTN7Ndw+xczalX2UpSdCf28wszlmlm5mH5tZ20TEWZqK63NMu6Fm5mZW7u/widJnM/t5+L2ebWb/KusYS1uEn+1UM5tsZt+GP9+nJSLO0mJmz5nZWjObtY/tZmZ/C78e6WbWt9SDcPdK/0Ewg+MioANQHZgJ9NirzbXAE+H784FXEx13nPt7HFA7fH9Nee5v1D6H7eoCnwFfA2mJjrsMvs+dgW+BA8PlpomOuwz6/BRwTfi+B7A00XGXsM9HA32BWfvYfhrBTLIGHAZMKe0YdEYS6A9kuvtid98NvAKcvVebs4EXw/djgBPMrLzOv1tsf919srtnh4tfA63LOMbSFuV7DHAPMBrYVZbBxUmUPl8FPObumwDcfW0Zx1jaovTZCabxBqgPrCrD+Eqdu39GMFX5vpwNvOSBr4EGZtaiNGNQIgm0AlbELGeF6wpt4+55wBagUZlEV/qi9DfWlQT/0ZRnxfY5POVv4+7vlmVgcRTl+9wF6GJmX5jZ12Y2qMyii48ofR4J/MLMsoAJwK/LJrSE2d/f9/1WtTQPJhWPmf0CSAOOSXQs8WRmVYA/AZclOJSyVpXg8taxBGedn5lZb3ffnNCo4usC4AV3f9jMBgL/MLNe7l6Q6MDKK52RBFYCbWKWW4frCm1jZlUJTok3lEl0pS9KfzGzE4HbgLPcPaeMYouX4vpcF+gFfGJmSwmuJY8v5wPuUb7PWcB4d8919yXAAoLEUl5F6fOVwGsA7v4VUJOgJlVFFen3vSSUSAJTgc5m1t7MqhMMpo/fq8144NLw/bnAJA9HssqhYvtrZocATxIkkfJ+3RyK6bO7b3H3xu7ezt3bEYwLneXu0xITbqmI8nP9JsHZCGbWmOBS1+KyDLKURenzcuAEADPrTpBI1pVplGVrPHBJePfWYcAWd19dmp9Al7YIxjzM7HpgIsFdH8+5+2wzGwVMc/fxwLMEp8CZBANb5ycu4pKJ2N8HgQOA18N7Cpa7+1kJC7qEIva5QonY54nAyWY2B8gHbnL38nqmHbXP/wc8bWa/Jxh4v6wc/1OImf2b4J+BxuG4z11ANQB3f4JgHOg0IBPIBi4v9RjK8ddPRESSgC5tiYhIiSiRiIhIiSiRiIhIiSiRiIhIiSiRiIhIiSiRSFIKq+8+HLN8o5mNLKVjv2Bm55bGsYr5PD8zs7lmNjnenyvRzOzWRMcgiaNEIskqBzgnfEguaYRVDaK6ErjK3Y+LVzxJRImkElMikWSVR1Du+/d7b9j7jMLMtoevx5rZp2b2lpktNrP7zewiM/vGzDLMrGPMYU40s2lmtsDMzgj3TzGzB81sajhvw9Uxx/2PmY0H5hQSzwXh8WeZ2ehw3Z3AkcCzZvZgIfsMD/eZaWb3h+sODgsnppvZG2Z2YLj+EzP7cxjvXDPrZ2bjzGyhmd0btmlnZvPM7J9hmzFmVjvcdoIFc29kWDB3RY1w/VIzu9vMZoTbuoXr64Ttvgn3Oztcf1n4ed8PP/cD4fr7gVpm9l34+euY2bth32aZ2Xn78X2X8ijRtfT1oY/CPoDtBKW+lxLUNbsRGBluewE4N7Zt+HossBloAdQgqCd0d7jtt8BfYvZ/n+Afqc4E9aZqAsOA28M2NYBpQPvwuDuA9oXE2ZKg5EYTgkoRk4DB4bZPKGROE+BU4Et+mO+lYfiaDhwTvh8VE+8nwOiYfqyK6WMWQRXqdgRPaR8Rtnsu/JrVJKj82iVc/xLwu/D9UuDX4ftrgWfC938EfhG+b0BQf6sOQUHLxeH3oyawjKBa8n+/B+H7ocDTMcv1E/3zpI/4fuiMRJKWu28l+MP3m/3Ybaq7r/agyOQi4INwfQbBH9s9XnP3AndfSPDHsRtwMkFNou+AKQR/oPcUMPzGg6KGe+sHfOLu6zyYXuCfBBMNFeVE4HkP53tx941mVh9o4O6fhm1e3Os4e0q4ZACzY/q4mB8K8q1w9y/C9y8TnBF1BZa4+4J9HHdc+DqdH74+JwMjwq/DJwRJIzXc9rEHdcl2EZydFTZzZgZwkpmNNrOj3H1LMV8PKedUa0uS3V+AGcDzMevyCC/LWlD+vXrMttgqxQUxywX8+Od979pATjCD3K/dfWLsBjM7luCMJJFi+7F3H/f0q7A+RT1ufsxxDBjq7vNjG5rZgL0+d+w+P3xS9wUWzO1yGnCvmX3s7qMixCLllM5IJKm5+0aCkt9XxqxeChwavj+LsEDdfvqZmVUJx006APMJCv1dY2bVAMysi5nVKeY43wDHmFljM0shmOvi02L2+RC4PGYMo2H4X/smMzsqbHNxhOPsLdWC+TUALgQ+D/vVzsw67cdxJwK/NguqdVpQCbo4uTFft5ZAtru/TFD8s/TnCJekojMSKQ8eBq6PWX4aeMvMZhKMdfyUs4XlBEmgHvArd99lZs8QXN6ZEf4RXQcMLuog7r7azEYAkwn+k3/X3d8qZp/3zexgYJqZ7SaoznorwTQFT4QJZjH7X6V1PnCdmT1HcNnp8bBflxNUca5KUGb9iWKOcw/BmWB6eMa3BDijmH2eCtvPILgc+aCZFQC5wDX72Q8pZ1T9V6QCMLN2wDvu3ivBoUglpEtbIiJSIjojERGREtEZiYiIlIgSiYiIlIgSiYiIlIgSiYiIlIgSiYiIlIgSiYiIlMj/A8IRA0yo0nYvAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "svd = TruncatedSVD()\n",
    "svd.fit(X_train)\n",
    "\n",
    "\n",
    "plt.plot(np.cumsum(svd.explained_variance_ratio_))\n",
    "plt.xlabel('Number of components')\n",
    "plt.ylabel('Cumulative explained variance')\n",
    "\n",
    "# svd = TruncatedSVD(n_components=500)\n",
    "# svd.fit(X_train)\n",
    "# X_train = svd.transform(X_train)\n",
    "# X_test = svd.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define learning curves plot function\n",
    "def plot_learning_curve(estimator, X, y, ylim=None, cv=None,\n",
    "                        n_jobs=1, train_sizes=np.linspace(.1, 1.0, 10), scoring='f1_macro'):\n",
    "\n",
    "    plt.figure()\n",
    "    \n",
    "    if ylim is not None:\n",
    "        plt.ylim(*ylim)\n",
    "    plt.xlabel(\"Training examples\")\n",
    "    plt.ylabel(\"Score\")\n",
    "    train_sizes, train_scores, test_scores = learning_curve(\n",
    "        estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes, \n",
    "        scoring= scoring)\n",
    "    train_scores_mean = np.mean(train_scores, axis=1)\n",
    "    train_scores_std = np.std(train_scores, axis=1)\n",
    "    test_scores_mean = np.mean(test_scores, axis=1)\n",
    "    test_scores_std = np.std(test_scores, axis=1)\n",
    "    plt.grid()\n",
    "\n",
    "    plt.fill_between(train_sizes, train_scores_mean - train_scores_std,\n",
    "                     train_scores_mean + train_scores_std, alpha=0.1,\n",
    "                     color=\"r\")\n",
    "    plt.fill_between(train_sizes, test_scores_mean - test_scores_std,\n",
    "                     test_scores_mean + test_scores_std, alpha=0.1, color=\"g\")\n",
    "    plt.plot(train_sizes, train_scores_mean, 'o-', color=\"r\",\n",
    "             label=\"Training score\")\n",
    "    plt.plot(train_sizes, test_scores_mean, 'o-', color=\"g\",\n",
    "             label=\"Validation score\")\n",
    "\n",
    "    plt.legend(loc=\"best\")\n",
    "    return plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train model function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, X_train, y_train):\n",
    "    model.fit(X_train, y_train) \n",
    "    \n",
    "    # Random permutation cross-validator\n",
    "    cv = ShuffleSplit(n_splits=3, test_size=0.2, random_state=0)\n",
    "    plot_learning_curve(model, X_train, y_train, (0.1, 1.01), cv=cv, n_jobs=2)\n",
    "\n",
    "    plt.show()\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate model function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, X_test, y_test):\n",
    "    y_pred = model.predict(X_test)\n",
    "    print(classification_report(y_test, y_pred.round()))  \n",
    "    print(\"Accuracy: \"+str(accuracy_score(y_test,  y_pred.round()))) \n",
    "    \n",
    "    precision_curve, recall_curve, _ = precision_recall_curve(y_test,  y_pred.round())\n",
    "    step_kwargs = ({'step': 'post'}\n",
    "               if 'step' in signature(plt.fill_between).parameters\n",
    "               else {})\n",
    "    plt.step(recall_curve, precision_curve, color='b', alpha=0.2,\n",
    "             where='post')\n",
    "    plt.fill_between(recall_curve, precision_curve, alpha=0.2, color='b', **step_kwargs)\n",
    "\n",
    "    plt.xlabel('Recall')\n",
    "    plt.ylabel('Precision')\n",
    "    plt.ylim([0.0, 1.05])\n",
    "    plt.xlim([0.0, 1.0])\n",
    "    plt.title('Precision-Recall curve')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GridSearchCV function for hyperparameter tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hyper_parameter_tuning(param_grid, model, cv):\n",
    "    #Set-up Grid Search and start fit\n",
    "    grid = GridSearchCV(estimator=model, param_grid=param_grid, cv=cv, verbose=1, n_jobs=10)\n",
    "    grid.fit(X_train, y_train)\n",
    "\n",
    "    #Print best option\n",
    "    print(\"Best options\")\n",
    "    print(\"=======================================\")\n",
    "    for param_name in sorted(param_grid.keys()):\n",
    "         print(\"%s: %r\" % (param_name, grid.best_params_[param_name]))\n",
    "            \n",
    "    return grid.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dummy classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xt8FPW9//HXhyRcw0VE0YomnB68hHsSQwVR8kA96KOVqihSrOItltZa609/ovVnrefQY09bi1qL4FFbNZqCl8qpKCqFKvWIXEQQEEEFRRAVKxjugc/vj50Mm81usgnZ3QTez8djyVy+M/PeybKfzOzOd8zdERERAWiV6QAiItJ8qCiIiEhIRUFEREIqCiIiElJREBGRkIqCiIiEVBRERCSkoiAiIqGUFQUze9jMPjOzdxLMNzO718zWmNlSMytMVRYREUlOdgrX/Ufg98CjCeafDfQKHoOAycHPOnXr1s3z8/ObJmE9tm3bRocOHdKyraak3OnTEjODcqdTc8m8aNGiL9z9iHobunvKHkA+8E6CeVOAMVHjq4Cj61tnUVGRp8ucOXPStq2mpNzp0xIzuyt3OjWXzMBCT+J9O5OfKRwDfBw1vj6YJiIiGWKewg7xzCwf+Ku794kz76/AXe4+LxifDdzs7gvjtC0DygC6d+9eVFFRkbLM0SorK8nNzU3LtpqScqdPS8wMyp1OzSVzaWnpIncvrq9dKj9TqM8nwLFR4z2CabW4+1RgKkBxcbEPGzYs5eEA5s6dS7q21ZSUO31aYmZQ7nRqaZkzWRRmANeaWQWRD5i3uPvGDOYROSTt2bOH9evXs3PnzkxHqVfnzp1ZuXJlpmM0SLozt23blh49epCTk9Oo5VNWFMzsSWAY0M3M1gM/B3IA3P0BYCZwDrAG2A5cnqosIpLY+vXr6dixI/n5+ZhZpuPU6euvv6Zjx46ZjtEg6czs7mzevJn169fTs2fPRq0jZUXB3cfUM9+BH6Vq+yKSnJ07d7aIgiD1MzMOP/xwPv/880avQ1c0i4gKwkHkQH+XKgoiIhJSURCRjNq8eTMDBgxgwIABHHXUURxzzDHh+O7du5Nax+WXX86qVavqbHP//fdTXl7eFJEPapn89pGItETl5fCzn8FHH8Fxx8HEiTB2bKNXd/jhh7NkyRIA7rjjDnJzc7nxxhtrtPH9PR/E9cgjj9S7nR/9qHl+hBleSdyqefyN3jxSiEjLUF4OZWWwbh24R36WlUWmN7E1a9ZQUFDA2LFj6d27Nxs3buS6666juLiY3r17c+edd4ZtTz31VJYsWUJVVRVdunRhwoQJ9O/fn1NOOYXPPvsMgNtuu41JkyaF7SdMmEBJSQknnHACr7/+OhDpp+iCCy6goKCAUaNGUVxcHBasaDfddBMFBQX069ePm2++GYBPP/2UkSNH0q9fP/r378/8+fMBmDRpEn369KFPnz7cd999CZ/bCy+8wCmnnEJhYSGjR49m27ZtTb5Pk6EjBRHZ7/rrIc6bYOiNN2DXrprTtm+HK6+EBx+Mv8yAARC8GTfUu+++y6OPPkpxceRC3DvuuIO8vDyqqqooLS1l1KhRFBQU1Fhmy5YtnH766dx1113ccMMNPPzww0yYMKHWut2dN998kxkzZnDnnXfy4osvct9993HUUUfx9NNP8/bbb1NYWLvz5k2bNjFz5kyWL1+OmfHVV18BkSORM888k2uvvZaqqiq2b9/O/PnzmTZtGgsWLKCqqoqSkhKGDRtGu3btajy3zz77jLvuuovZs2fTvn17Jk6cyD333MOtt97aqP12IHSkICLJiy0I9U0/QN/85jfDggDw1FNPUVhYSGFhIStXrmTFihW1lmnXrh1nn302AEVFRaxduzbuus8///xabebNm8fFF18MQP/+/endu3et5bp27UqrVq24+uqrefbZZ8MeUOfOncs111wDQHZ2Np06dWLevHmce+65tGvXjo4dO/Ld736X1157rdZze/3111mxYgWDBw9mwIABlJeXJ8ydajpSEJH96vuLPj8/csooVl4ezJ3b5HGiu5xevXo1kydPZuHChXTp0oVLLrkk7lXYrVu3DoezsrKoqqqKu+42bdrU2yaenJwcFi5cyMsvv8z06dOZPHkyL730EtCwr4NGPzd3Z8SIETz22GNJL58qOlIQkeRNnAjt29ec1r59ZHqKbd26lY4dO9KpUyc2btzIrFmzmnwbQ4YMYdq0aQAsW7Ys7pHI119/zdatW/n2t7/N7373O9566y0ASktLeeCBBwDYu3cvW7duZejQofz1r39lx44dVFZW8txzzzF06NBa6xw8eDB///vf+eCDD4DIZxurV69u8ueXDB0piEjyqr9l1ITfPkpWYWEhJ5xwAieeeCJ5eXkMGTKkybfx4x//mEsvvZSCgoLw0blz5xpttmzZwvnnn8+uXbvYt28fd999NwC///3vufrqq5kyZQrZ2dlMmTKFkpISRo0axcknnwzA+PHj6du3L2vWrKmxzu7du/PQQw8xevTo8Gu4v/zlL+nVq1eTP8d6JXPTheb00E126qfc6dMSM7vXzL1ixYrMBWmgrVu3pnT9e/bs8R07dri7+3vvvef5+fm+Z8+eA1pnqjPHE+93SpI32dGRgohIoLKykuHDh1NVVYW7h3/1H0oOrWcrIlKHLl26sGjRokzHyCh90CwiIiEVBRERCakoiIhISEVBRERCKgoiklGlpaW1LkSbNGkS48ePr3O53NxcADZs2MCoUaPithk2bBgLFy6scz2TJk1i+/bt4fg555wT9md0KFJREJEGKV9WTv6kfFr9ohX5k/IpX3ZgPaSOGTOGioqKGtMqKioYM6bOO/qGvvGNb/DUU081evuxRWHmzJl06dKl0etLlb1796ZlOyoKIpK08mXllP1PGeu2rMNx1m1ZR9n/lB1QYRg1ahTPP/98eCXv2rVr2bBhA0OHDg2vGygsLKRv3748//zztZZfu3Ytffr0AWDHjh1cfPHFnHTSSZx33nns2LEjbDd+/Piw2+2f//znANx7771s2LCB0tJSSktLAcjPz+eLL74A4O677w67va7udnvt2rWcdNJJXH311fTu3ZuzzjqrxnaqTZ8+nT59+jB48GBOO+00IPLGfuONN9KnTx/69esXdqU9e/ZsBg4cSN++fbniiivYFXQwmJ+fz80330xhYSHTp0/n/fffZ8SIERQVFTF06FDefffdRu/3RHSdgoiErn/xepZ8mrjr7DfWv8GuvTV7RN2+ZztXPnclDy6K33X2gKMGMGlE4o72unbtSklJCS+88AIjR46koqKCiy66CDOjbdu2PPvss3Tq1IkvvviCkpISRo8enbDjucmTJ9O+fXtWrlzJ0qVLa3R9PXHiRLp27crevXsZPnw4S5cu5brrruPuu+9mzpw5dOvWrca6Fi1axCOPPML8+fNxdwYNGsTpp5/OYYcdxurVq3nyySd58MEHueiii3j66ae55JJLaix/5513MmvWLDp16hT+lT916lTWrl3LkiVLyM7O5ssvv2Tnzp2MGzeO2bNnc/zxx3PppZcyefJkrr/+eiByE6LFixcDMHz4cB544AF69erF/Pnz+eEPf8jf/va3hPu2MXSkICJJiy0I9U1PVvQppOhTR+7OrbfeSr9+/TjjjDPYuHEjmzZtSrieV199NXxz7tevH/369QvnTZs2jcLCQgYOHMjy5cvjdnYXbd68eZx33nl06NCB3Nxczj///LDb6549ezJgwAAgcffcQ4YMYdy4cfzxj38Mi8Irr7zCNddcE14l3bVrV1atWkXPnj05/vjjAbjssst49dVXw/WMHj0aiFxt/frrr3PhhRcyYMAArrnmGjZu3Fjnc2gMHSmISKiuv+gB8ifls25L7a6z8zrnMXfc3EZvd+TIkfz0pz9l8eLFbN++naKiIgDKy8v5/PPPWbRoETk5OeTl5cXtLrs+H374Ib/5zW9YsGABhx12GOPGjWvUeqpVd7sNka63450+euCBB5g/fz7PPPMMRUVFjb5SurqL7X379tGlS5e4d4JrSjpSEJGkTRw+kfY5NbvObp/TnonDD6zr7NzcXEpLS7niiitqfMC8ZcsWjjzySHJycpgzZw4fffRRnes57bTTeOKJJwB45513WLp0KRDpdrtDhw507tyZTZs28cILL4TLdOzYka+//rrWuoYOHcpf/vIXtm/fzrZt23j22WfjdnudyPvvv8+gQYO47bbbOOKII/j4448588wzmTJlSnj/hi+//JITTjiBtWvXhj2nPvbYY5x++um11tepUyd69uzJ9OnTgchR1Ntvv510nmSpKIhI0sb2HcvU70wlr3MehpHXOY+p35nK2L4H3nX2mDFjePvtt2sUhbFjx7Jw4UL69u3Lo48+Gp5iSWT8+PFUVlZy0kkncfvtt4dHHP3792fgwIGceOKJfO9736vR7XZZWRkjRowIP2iuVlhYyLhx4ygpKWHQoEFcddVVDBw4MOnnc9NNN9G3b18GDRrE4MGD6d+/P1dddRXHHXdceB/nJ554grZt2/LII49w4YUX0rdvX1q1asUPfvCDuOssLy/noYceCu8K99xzzyWdJ2nJdKXanB7qOrt+yp0+LTGzu7rOTqeW1nW2jhRERCSkoiAiIiEVBREhcnZBDgYH+rtUURA5xLVt25bNmzerMBwE3J3NmzfTtm3bRq9D1ymIHOJ69OjB+vXr+fzzzzMdpV47d+48oDe8TEh35rZt29KjR49GL6+iIHKIy8nJoWfPnpmOkZS5c+c26GuhzUFLy6zTRyIiElJREBGRkIqCiIiEVBRERCSkoiAiIiEVBRERCakoiIhIKKVFwcxGmNkqM1tjZhPizD/OzOaY2VtmttTMzkllHhERqVvKioKZZQH3A2cDBcAYMyuIaXYbMM3dBwIXA39IVR4REalfKo8USoA17v6Bu+8GKoCRMW0c6BQMdwY2pDCPiIjUI5XdXBwDfBw1vh4YFNPmDuAlM/sx0AE4I4V5RESkHpaqnhHNbBQwwt2vCsa/Dwxy92uj2twQZPitmZ0CPAT0cfd9MesqA8oAunfvXlRRUZGSzLEqKyvJzc1Ny7aaknKnT0vMDMqdTs0lc2lp6SJ3L663YTK3Z2vMAzgFmBU1fgtwS0yb5cCxUeMfAEfWtV7djrN+yp0+LTGzu3KnU3PJTDO4HecCoJeZ9TSz1kQ+SJ4R0+YjYDiAmZ0EtAWaf/+9IiIHqZQVBXevAq4FZgEriXzLaLmZ3Wlm5wbN/g9wtZm9DTwJjAsqmoiIZEBK76fg7jOBmTHTbo8aXgEMSWUGERFJnq5oFhGRkIqCiIiEVBRERCSkoiAiIiEVBRERCakoiIhISEVBRERCKgoiIhJSURARkZCKgoiIhFQUREQkpKIgIiIhFQUREQmpKIiISEhFQUREQioKIiISUlEQEZGQioKIiIRUFEREJKSiICIiIRUFEREJqSiIiEhIRUFEREIqCiIiElJREBGRkIqCiIiEVBRERCSkoiAiIiEVBRERCakoiIhISEVBRERCKgoiIhJSURARkZCKgoiIhFQUREQkpKIgIiIhFQUREQmpKIiISEhFQUREQiktCmY2wsxWmdkaM5uQoM1FZrbCzJab2ROpzCMiInXLTtWKzSwLuB84E1gPLDCzGe6+IqpNL+AWYIi7/9PMjkxVHhERqV/SRwpmdqqZXR4MH2FmPetZpARY4+4fuPtuoAIYGdPmauB+d/8ngLt/lnx0ERFpaubu9Tcy+zlQDJzg7seb2TeA6e4+pI5lRgEj3P2qYPz7wCB3vzaqzV+A94AhQBZwh7u/GGddZUAZQPfu3YsqKioa8BQbr7Kyktzc3LRsqykpd/q0xMyg3OnUXDKXlpYucvfi+tole/roPGAgsBjA3TeYWccDyBe9/V7AMKAH8KqZ9XX3r6IbuftUYCpAcXGxDxs2rAk2Xb+5c+eSrm01JeVOn5aYGZQ7nVpa5mRPH+32yCGFA5hZhySW+QQ4Nmq8RzAt2npghrvvcfcPiRw19Eoyk4iINLFki8I0M5sCdDGzq4FXgAfrWWYB0MvMeppZa+BiYEZMm78QOUrAzLoBxwMfJJlJRESaWFKnj9z9N2Z2JrAVOAG43d1frmeZKjO7FphF5POCh919uZndCSx09xnBvLPMbAWwF7jJ3TcfwPMREZEDUG9RCL5a+oq7lwJ1FoJY7j4TmBkz7faoYQduCB4iIpJh9Z4+cve9wD4z65yGPCIikkHJfvuoElhmZi8D26onuvt1KUklIiIZkWxReCZ4iIjIQSzZD5r/FHyD6Phg0ip335O6WCIikglJFQUzGwb8CVgLGHCsmV3m7q+mLpqIiKRbsqePfguc5e6rAMzseOBJoChVwUREJP2SvXgtp7ogALj7e0BOaiKJiEimJHuksNDM/ht4PBgfCyxMTSQREcmUZIvCeOBHQPVXUF8D/pCSRCIikjHJFoVs4B53vxvCq5zbpCyViIhkRLKfKcwG2kWNtyPSKZ6IiBxEki0Kbd29snokGG6fmkgiIpIpyRaFbWZWWD1iZsXAjtREEhGRTEn2M4XrgelmtiEYPxoYnZpIIiKSKXUeKZjZyWZ2lLsvAE4E/gzsAV4EPkxDPhERSaP6Th9NAXYHw6cAtwL3A/8kuGeyiIgcPOo7fZTl7l8Gw6OBqe7+NPC0mS1JbTQREUm3+o4UssysunAMB/4WNS/ZzyNERKSFqO+N/Ung72b2BZFvG70GYGb/CmxJcTYREUmzOouCu080s9lEvm30UnBPZYgcYfw41eFERCS96j0F5O5vxJn2XmriiIhIJiV78ZqIiBwCVBRERCSkoiAiIiEVBRERCakoiIhI6JAoCuXLysmflE+rX7Qif1I+5cvKM5Nj8g/JvymbVncY+TdlUz75h8oRlePN2ZMynqG57AvlUI5M5Tjoi0L5snLKnr2CdVvW4Tjrtqyj7Nkr0l4Yyif/kLJPJrMudy9usC53L2WfTE77i6w55/jFvufSmqM57wvlUI5M5bD916O1DMXFxb5w4cKk2+dP7Ma6qs21prfxVgzMPT4Y8xo/qkd279xD6zY5NabFaxc7aNUjUbMWb3+fXXGuCmlTBYWt88Ad8+rlqh/R42Du+9db3Yb9wxaM792zl6wsq7U87izo9HXCHCX/rL5vksc8RxJPD7ZptZrGWYGH/zD/yD0Jcwz6rHXtFVrNCW61tlirTWQdlnD8zS7bEu+LLR0S7IPgOQTz9u1zWrWKzVLH/ov6EeY4fGfiffFF1F1vo/ZfolhxNxBnvnvM7nKYf1RV4hwbs+KtKOY1n3h+giBx57/Rg4Q5vrWemr/HqGEn3rxgoEbAmPEaO2L/vPnddtX/f8XivL6itusG+/Y6rbJb7W8UL1eN9ViN+QvabI6bI68yi7W/rqo9IwEzW+TuxfW1O+j7L/poz+Y471iwi320Wf5uncs25U2od+UlmJ4FrT9Yl9xKghedm9V8AVa/iKJ+WvS0qLa7sohrVxZ4VvSBY+x/rJggMdPD/9sJ3qxj/+/vyvoqYY69ue0jq0/0B0uiN8dEb4oJpte5L2q9cRB3xFslKlDx112rgcGurJ0Jc+xr2zap9cXNUCvL/jZ790Hsr3tX1j8T5tjbtXO8FSbYToI2UZM8dn7U6K6szxPm2HlEN8L31+jfa62XQ+y0xH/Q1Nh81GtrV9auhDk8O6vWOnCw8I82gH2R/577HNuzLyZD/OFar3t3dh0dNwYfddgbf8YBOuiLwnFbYF2X2tPztsDcU/8bWrUK3jij30Qjj7c+/ZSB3/hGrenA/uVi58W2M4OsLPJnf4d1nfbVzlHZirnj/xeysiA7O/IzKwtycvYPZ2dH1tMq6n9x9XCcaXMXL2ZYcXHt+Wbk39GFdR1rv5jyKrN47T83Jd6Rdb3pNLSdGfm3dWBdbpwc27KY98tPG77dRrTPv7Vdwgyv/fbLmhMTFKi5//gHwwYPrj0jYUGrPT3//3VM/DuZGNzXKsn9mmybua+/Xit3nb+TiRvrX3ey6jk7UVeOu86ZxrAhQxIv3ISv0/xb2iZ+ffzmyzhLBGKe39x58xh26qkJ59e3jkT747htCf6qOUAHfVGYuORwygZvZnvr/dPa74aJb3WFX36v9l/ZUcNb/vEPiP5l1tG2zmFg4rprKPtkMtujzka13wMTv3kNlJQ05qkllpUFnTvHnTXxm2UJcpRB+/Tddnviv8TJsTsynTZNeYzWwAx7ggw5OYkXjGZ2wHkz8juJk7vO/dG6NelSZw6ztGWpM0d2A946zZJ/PTU0Rwoc9B80j73qHqbOyiHvKzCHvK9g6qwcxl59L7RrB23bRh5t2ux/tG4deVT/Mqsf2dk1/5rPyor8JV79iD1KiM4x/g9MPWY8eZVZkRyVWUw9Zjxjx/8hvfujGef4eauRac3RnPeFcihHxnK4e4t6FBUVeYM9/rh7Xp67WeTn448ntdicOXMavq1mQLnTpyVmdlfudGoumYGFnsR77EF/+giAsWMjDxERqdNBf/pIRESSp6IgIiIhFQUREQmltCiY2QgzW2Vma8xsQh3tLjAzN7N6r7YTEZHUSVlRMLMs4H7gbKAAGGNmBXHadQR+AsxPVRYREUlOKo8USoA17v6Bu+8GKoCRcdr9O/ArIP61/iIikjapLArHAB9Hja8PpoXMrBA41t2fT2EOERFJUsauUzCzVsDdwLgk2pYBZQDdu3dn7ty5Kc1WrbKyMm3bakrKnT4tMTModzq1uMzJXOHWmAdwCjAravwW4Jao8c7AF8Da4LET2AAU17XeRl3R3EjN5UrEhlLu9GmJmd2VO52aS2aSvKI5laePFgC9zKynmbUGLgZmRBWjLe7ezd3z3T0feAM4192Tv1mCiIg0qZQVBXevAq4FZgErgWnuvtzM7jSzc1O1XRERabyUfqbg7jOBmTHTbk/Qdlgqs4iISP10RbOIiIRUFEREJKSiICIiIRUFEREJqSiIiEhIRUFEREIqCiIiElJREBGRkIqCiIiEVBRERCSkoiAiIiEVBRERCakoiIhISEVBRERCKgoiIhJSURARkZCKgoiIhFQUREQkpKIgIiIhFQUREQmpKIiISEhFQUREQioKIiISUlEQEZGQioKIiIRUFEREJKSiICIiIRUFEREJqSiIiEhIRUFEREIqCiIiElJREBGRkIqCiIiEVBRERCSkoiAiIiEVBRERCakoiIhISEVBRERCKgoiIhJKaVEwsxFmtsrM1pjZhDjzbzCzFWa21Mxmm1leKvOIiEjdUlYUzCwLuB84GygAxphZQUyzt4Bid+8HPAX8V6ryiIhI/VJ5pFACrHH3D9x9N1ABjIxu4O5z3H17MPoG0COFeUREpB6pLArHAB9Hja8PpiVyJfBCCvOIiEg9zN1Ts2KzUcAId78qGP8+MMjdr43T9hLgWuB0d98VZ34ZUAbQvXv3ooqKipRkjlVZWUlubm5attWUlDt9WmJmUO50ai6ZS0tLF7l7cb0N3T0lD+AUYFbU+C3ALXHanQGsBI5MZr1FRUWeLnPmzEnbtpqScqdPS8zsrtzp1FwyAws9iffYVJ4+WgD0MrOeZtYauBiYEd3AzAYCU4Bz3f2zFGYREZEkpKwouHsVkVNCs4gcCUxz9+VmdqeZnRs0+zWQC0w3syVmNiPB6kREJA2yU7lyd58JzIyZdnvU8Bmp3L6IiDSMrmgWEZGQioKIiIRUFEREJKSiICIiIRUFEREJqSiIiEhIRUFEREIqCiIiElJREBGRkIqCiIiEVBRERCSkoiAiIiEVBRERCakoiIhISEVBRERCKgoiIhJSURARkZCKgoiIhFQUREQkZO6e6QwNYmafA+vStLluwBdp2lZTUu70aYmZQbnTqblkznP3I+pr1OKKQjqZ2UJ3L850joZS7vRpiZlBudOppWXW6SMREQmpKIiISEhFoW5TMx2gkZQ7fVpiZlDudGpRmfWZgoiIhHSkICIioUO2KJjZsWY2x8xWmNlyM/tJMP0OM/vEzJYEj3OilrnFzNaY2Soz+7cMZl9rZsuCfAuDaV3N7GUzWx38PCyYbmZ2b5B7qZkVZijzCVH7dImZbTWz65vj/jazh83sMzN7J2pag/evmV0WtF9tZpdlIPOvzezdINezZtYlmJ5vZjui9vkDUcsUBa+tNcHzsgzkbvBrwsxGBNPWmNmEVGauI/efozKvNbMlwfRms7+T4u6H5AM4GigMhjsC7wEFwB3AjXHaFwBvA22AnsD7QFaGsq8FusVM+y9gQjA8AfhVMHwO8AJgwLeA+c1g32cBnwJ5zXF/A6cBhcA7jd2/QFfgg+DnYcHwYWnOfBaQHQz/KipzfnS7mPW8GTwPC57X2RnY1w16TQSP94F/AVoHbQrSnTtm/m+B25vb/k7mccgeKbj7RndfHAx/DawEjqljkZFAhbvvcvcPgTVASeqTJm0k8Kdg+E/Ad6OmP+oRbwBdzOzoTASMMhx4393ruggxY/vb3V8FvoyTpyH799+Al939S3f/J/AyMCKdmd39JXevCkbfAHrUtY4gdyd3f8Mj71iPsv95pkSCfZ1IotdECbDG3T9w991ARdA2ZerKHfy1fxHwZF3ryMT+TsYhWxSimVk+MBCYH0y6Njjkfrj6NAGRgvFx1GLrqbuIpJIDL5nZIjMrC6Z1d/eNwfCnQPdguDnlrnYxNf/DNPf9DQ3fv80t/xVE/hKt1tPM3jKzv5vZ0GDaMURyVstk5oa8Jprbvh4KbHL31VHTmvv+Dh3yRcHMcoGngevdfSswGfgmMADYSOQwsLk51d0LgbOBH5nZadEzg786muXXysysNXAuMD2Y1BL2dw3Nef/GY2Y/A6qA8mDSRuA4dx8I3AA8YWadMpUvjhb3mogxhpp/9DT3/V3DIV0UzCyHSEEod/dnANx9k7vvdfd9wIPsP2XxCXBs1OI9gmlp5+6fBD8/A54lknFT9Wmh4OdnQfNmkztwNrDY3TdBy9jfgYbu32aR38zGAd8GxgbFjOD0y+ZgeBGR8/HHB/miTzFlJHMjXhPNYl8DmFk2cD7w5+ppzX1/xzpki0Jw3u8hYKW73x01Pfp8+3lA9bfYS/OGAAAFE0lEQVQLZgAXm1kbM+sJ9CLyIVFamVkHM+tYPUzkw8R3gnzV33C5DHguGJ4BXBp8S+ZbwJao0yCZUOOvqOa+v6M0dP/OAs4ys8OC0x9nBdPSxsxGAP8XONfdt0dNP8LMsoLhfyGybz8Icm81s28F/z8uZf/zTGfuhr4mFgC9zKxncCR6cdA2E84A3nX38LRQc9/ftWT6k+5MPYBTiZwCWAosCR7nAI8By4LpM4Cjo5b5GZEqv4oMfUuAyDcs3g4ey4GfBdMPB2YDq4FXgK7BdAPuD3IvA4ozuM87AJuBzlHTmt3+JlK0NgJ7iJznvbIx+5fIefw1wePyDGReQ+Rce/Xr+4Gg7QXBa2cJsBj4TtR6iom8Cb8P/J7gAtc0527wayL4v/teMO9nmXiNBNP/CPwgpm2z2d/JPHRFs4iIhA7Z00ciIlKbioKIiIRUFEREJKSiICIiIRUFEREJqShIs2Nmh0f1KPlpTI+ZrZNcxyNmdkI9bX5kZmObJnXzYGbzzGxApnNIy6WvpEqzZmZ3AJXu/puY6Ubk9bsvI8GaKTObB1zr7ksynUVaJh0pSIthZv9qkftflBO5GOhoM5tqZgstck+M26PazjOzAWaWbWZfmdldZva2mf2vmR0ZtPkPM7s+qv1dZvamRfrlHxxM72BmTwfbfSrYVq2/xM3s5KCzs0Vm9oKZdTeznGD81KDNr83sF8HwL8xsgZm9Y2YPBEWuOsfdwXZWmFmxRe6FsDookNX7YbmZVZjZSjObZmbt4mQ6O3i+iy3S13+HqBwrLNLh3K+a9JckLZ6KgrQ0JwK/c/cCj/QBNcHdi4H+wJlmVhBnmc7A3929P/C/RK40jsfcvQS4CaguMD8GPnX3AuDfifSmW3MhszbAPcAF7l4EPA78u7vvAS4HpprZWUAp8B/BYve4+8lA3yBfdLfaO4Ln9BDwF+AHQbsyC26UQ+TeApPc/SRgJ3BNTKYjidz3YbhHOk9cCvzEzLoTufq3t7v3A/4zwb6QQ5SKgrQ077v7wqjxMWa2mEj3AScRebOMtcPdq7uNXkTkpifxPBOnzalE+ufH3au7Fol1EtAbeMUid9uaQNBBm7svDZZ/DrgiKBQAw83sTSLdlZweLF+tut+eZcAyj3QQt5PIzZWqO1D70CP3b4BIETo1JtNgIvvi9SDT2OA5fQnsAx40s/OAbQn2hRyisjMdQKSBwjcxM+sF/AQocfevzOxxoG2cZXZHDe8l8et+VxJt4jFgqbsPTTC/D7AFqD5t1Z5IPzeF7v6Jmf1HTO7qHPuihqvHq3PFfhgYO27Ai+7+/VphzYqBM4ELgfFEOusTAXSkIC1bJ+BrIj1NVt/trKn9g8hdtDCzvsQ/ElkBHGNmJUG71mbWOxgeDeQCw4D7LdKPfjsib/BfWKTH2wsakaunmZ0cDH8PmBcz/3Xg9KBXzurPRnoF2+vk7n8Ffkqc02FyaNORgrRki4m8Ib8LrCPyBt7U7gMeNbMVwbZWEPmrP+Tuu8xsFHBv8KafBfzWzD4n8jnEMHffYGZTiHwecqWZ/SlY10b23/GvIVYCNwQfei8DpsZk2mRmVwJ/jvoa763ADuCZ4HOQVkRu+iIS0ldSRepgkZumZLv7zuB01UtAL99/7+NMZPpX4Cl31/UI0uR0pCBSt1xgdlAcDLgmkwVBJNV0pCAiIiF90CwiIiEVBRERCakoiIhISEVBRERCKgoiIhJSURARkdD/B3T7I/qHrwspAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dummy = DummyClassifier(strategy='most_frequent')  \n",
    "train(dummy, X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.84      1.00      0.91       484\n",
      "           1       0.00      0.00      0.00        95\n",
      "\n",
      "   micro avg       0.84      0.84      0.84       579\n",
      "   macro avg       0.42      0.50      0.46       579\n",
      "weighted avg       0.70      0.84      0.76       579\n",
      "\n",
      "Accuracy: 0.8359240069084629\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/spiros/env/lib/python3.5/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAF0lJREFUeJzt3X20XXV95/H3x/CkggGNWE0iQQNV6hN6iziupXRUilSDy/pARrQ4lLRWWp1aW2fVqZRWrTrSsSNWqVAUFQTHutKKpT6gjFY0lwEpD0IjIiQwg2gIIsjjd/7Y+5LjJdn35Hr3Pecm79dad3H2Pr+z93f/uDmfu/fvnN9OVSFJ0rY8ZNQFSJLGm0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBoQUlyRZLDZmjz+CS3J1k0T2X1Lsl1SV7YPj4xySdGXZN2HgaF5kT7RnZn+wb9/5KckWTPud5PVf1KVX11hjbXV9WeVXXfXO+/fZO+pz3OW5P8a5LnzPV+pHFiUGguvbSq9gSeCUwAb5/eII2F/nv36fY4lwAXAOeOuJ45l2SXUdeg8bHQ/8FqDFXVRuALwFMAknw1yTuTfAO4A3hCksVJTktyU5KNSf5y8FJRkuOTXJXkJ0muTPLMdv3gJZhDkkwmua09izm5Xb8iSU292SV5XJK1SX6cZH2S4wf2c2KSc5J8vN3XFUkmhjzOe4FPAkuTPHpgmy9JcunAGcfTBp5bnuSzSX6Y5EdJPtiuf2KSr7TrbknyySR7z6b/kxzV7v+2JN9LcsT0vhs49k9M67PjklwPfCXJF5KcMG3b30ny8vbxk5J8se3Xq5O8ajb1avwZFJpzSZYDRwKXDKx+LbAG2Av4AXAGcC+wEjgYOBz47fb1rwROBF4HPAJYBfxoK7v6APCBqnoE8ETgnG2UdDawAXgc8ArgXUn+48Dzq9o2ewNrgQ8OeZy7tTX+CNjUrjsYOB34HeBRwEeAtUl2b4Pwn9rjXwEsbfcLEODdbY1PBpa3fbBdkhwCfBx4a3s8zwOu245NPL/d/68DZwGrB7Z9ELAf8PkkDwe+CHwK2Bc4GvhQ20Y7GINCc+lzSW4Fvg58DXjXwHNnVNUV7V/hj6QJkjdX1U+r6mbgr2nebKAJjPdW1bpqrK+qH2xlf/cAK5Msqarbq+qi6Q3a0Hou8CdV9bOquhT4KM0b/JSvV9V57ZjGmcDTZzjOV7XHeSdwPPCK9rigCcOPVNW3quq+qvoYcBdwKHAITRC8tT3un1XV1wHaY/xiVd1VVT8ETqZ5095exwGnt9u6v6o2VtV3t+P1J7a13Qn8A/CMJPu1z70G+GxV3QW8BLiuqv6+qu6tqkuA/wW8chY1a8wZFJpLL6uqvatqv6r6vfbNZsoNA4/3A3YFbmovz9xK85f3vu3zy4HvDbG/44ADge8mWZfkJVtp8zjgx1X1k4F1P6D5a37K/x14fAewR5JdkrymHbS+PckXBtqcU1V7A48BLgeeNe3Y3jJ1XO2xLW/rWA78YCBUHpDkMUnObi/D3QZ8gmYMZHsN23fb8sD/p7bPPs+WAF9Nc6kNmuN89rTjfA3wS7/AvjWmHLDSfBmcpvgGmr+yl2ztTbN9/okzbrDq34HV7eD4y4HPJHnUtGY3Ao9MstdAWDwe2DjE9j/JljfGrT1/S5I1wGSST1XVTW3t76yqd05v33466vFJdtnKcb+Lpo+eWlU/TvIyhrwENk1X3/0UeNjA8tbe1KdPJ30W8I4kFwJ70AzeT+3na1X1olnUqAXGMwrNu/YN9V+A9yd5RJKHtIO5U5daPgr8UZJnNR+SysqByx8PSHJMkkdX1f3Are3q+6ft6wbgX4F3J9mjHVg+juYv9rk4lquB84E/blf9HfC7SZ7d1v7wJL+RZC/g28BNwF+16/dI8tz2dXsBtwObkyylGWOYjdOA1yd5QduvS5M8qX3uUuDoJLu2A/avGGJ759GcPZxE82mvqf79J+DAJK9tt7drkl9N8uRZ1q0xZlBoVF4H7AZcSTMQ/BngsQBVdS7wTpqB0p8An6MZ15juCOCKJLfTDGwfPe1y15TVNIPHN9Jcd39HVX1pDo/lfcCaJPtW1STNuMUH2+NaDxwL0I6BvJRmAP96mgH2V7fb+HOajxVvprnc89nZFFJV3wZeTzPms5lmrGgqZP8bzdnGpnZ/nxpie3e1tbxwsH17dnY4zWWpG2ku370H2H02dWu8xRsXSZK6eEYhSepkUEiSOhkUkqROBoUkqdOC+x7FkiVLasWKFaMuQ5IWlIsvvviWqnr0zC0fbMEFxYoVK5icnBx1GZK0oCTZ2jQ4Q/HSkySpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnq1FtQJDk9yc1JLt/G80nyN2nuYXxZ2nsiS5LGS59nFGfQTAO9LS8GDmh/1gB/22MtkqRZ6i0oqupC4McdTY4CPt7eE/kiYO8kj51pu3ffPVcVSpKGMcoxiqX8/H2UN/Dz9zF+QJI1SSaTTN5006Z5KU6S1FgQg9lVdWpVTVTVxOLF+4y6HEnaqYwyKDYCyweWlzHEDe8lSfNrlEGxFnhd++mnQ4HNVXXTCOuRJG1Fb7PHJjkLOAxYkmQD8A5gV4Cq+jBwHnAkzc3n76C5Ibwkacz0FhRVtXqG5wt4Y1/7lyTNjQUxmC1JGh2DQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ16DYokRyS5Osn6JG/byvOPT3JBkkuSXJbkyD7rkSRtv96CIski4BTgxcBBwOokB01r9nbgnKo6GDga+FBf9UiSZqfPM4pDgPVVdW1V3Q2cDRw1rU0Bj2gfLwZu7LEeSdIs9BkUS4EbBpY3tOsGnQgck2QDcB7w+1vbUJI1SSaTTG7evKmPWiVJ2zDqwezVwBlVtQw4EjgzyYNqqqpTq2qiqiYWL95n3ouUpJ1Zn0GxEVg+sLysXTfoOOAcgKr6JrAHsKTHmiRJ26nPoFgHHJBk/yS70QxWr53W5nrgBQBJnkwTFD/ssSZJ0nbqLSiq6l7gBOB84CqaTzddkeSkJKvaZm8Bjk/yHeAs4Niqqr5qkiRtvyy09+WVKydq/frJUZchSQtKkouramI2rx31YLYkacwZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSp0y7DNkyyFNhv8DVVdWEfRUmSxsdQQZHkPcCrgSuB+9rVBXQGRZIjgA8Ai4CPVtVfbaXNq4AT2+19p6r+07DFS5L6N+wZxcuAX66qu4bdcJJFwCnAi4ANwLoka6vqyoE2BwD/FXhuVW1Ksu/wpUuS5sOwYxTXArtu57YPAdZX1bVVdTdwNnDUtDbHA6dU1SaAqrp5O/chSerZsGcUdwCXJvky8MBZRVX9QcdrlgI3DCxvAJ49rc2BAEm+QXN56sSq+ucha5IkzYNhg2Jt+9PH/g8ADgOWARcmeWpV3TrYKMkaYA3AkiVP6KEMSdK2DBUUVfWxJLvRngEAV1fVPTO8bCOwfGB5Wbtu0AbgW+22vp/kGprgWDdt/6cCpwKsXDlRw9QsSZobQ41RJDkM+HeawekPAdcked4ML1sHHJBk/zZkjubBZyWfozmbIMkSmiC6dtjiJUn9G/bS0/uBw6vqaoAkBwJnAc/a1guq6t4kJwDn04w/nF5VVyQ5CZisqrXtc4cnmfrY7Vur6kezPxxJ0lxL1cxXcpJcVlVPm2ndfFi5cqLWr5+c791K0oKW5OKqmpjNa4c9o5hM8lHgE+3yawDfrSVpJzBsULwBeCMw9XHY/00zViFJ2sEN+6mnu4CT2x9J0k6kMyiSnFNVr0rybzRzMf2cUYxRSJLm10xnFG9q//uSvguRJI2nzu9RVNVN7cNbgBuq6gfA7sDTgRt7rk2SNAaGnRTwQmCP9p4U/wK8Fjijr6IkSeNj2KBIVd0BvBz4UFW9EviV/sqSJI2LoYMiyXNovj/x+Xbdon5KkiSNk2GD4s00Nxj6h3YajicAF/RXliRpXAz7PYqvAV8bWL6WLV++kyTtwGb6HsX/qKo3J/lHtv49ilW9VSZJGgsznVGc2f73v/ddiCRpPHUGRVVd3D6cBO6sqvsBkiyi+T6FJGkHN+xg9peBhw0sPxT40tyXI0kaN8MGxR5VdfvUQvv4YR3tJUk7iGGD4qdJnjm1kORZwJ39lCRJGifD3o/izcC5SW4EAvwS8OreqpIkjY1hv0exLsmTgF9uV11dVff0V5YkaVwMdekpycOAPwHeVFWXAyuSOPW4JO0Ehh2j+HvgbuA57fJG4C97qUiSNFaGDYonVtV7gXsA2plk01tVkqSxMWxQ3J3kobTTeCR5InBXb1VJksbGsJ96egfwz8DyJJ8Engsc21dRkqTxMWNQJAnwXZqbFh1Kc8npTVV1S8+1SZLGwIxBUVWV5LyqeipbblokSdpJDDtG8X+S/GqvlUiSxtKwYxTPBo5Jch3wU5rLT1VVT+urMEnSeBg2KH691yokSWNrpjvc7QH8LrAS+DfgtKq6dz4KkySNh5nGKD4GTNCExIuB9/dekSRprMx06emg9tNOJDkN+Hb/JUmSxslMZxQPzBDrJSdJ2jnNFBRPT3Jb+/MT4GlTj5PcNtPGkxyR5Ook65O8raPdbyapJBPbewCSpH51XnqqqkWz3XCSRcApwIuADcC6JGur6spp7fYC3gR8a7b7kiT1Z9gv3M3GIcD6qrq2qu4GzgaO2kq7vwDeA/ysx1okSbPUZ1AsBW4YWN7QrntAex/u5VXVOTVIkjVJJpNMbt68ae4rlSRtU59B0SnJQ4CTgbfM1LaqTq2qiaqaWLx4n/6LkyQ9oM+g2AgsH1he1q6bshfwFOCr7dQghwJrHdCWpPHSZ1CsAw5Isn+S3YCjgbVTT1bV5qpaUlUrqmoFcBGwqqome6xJkrSdeguK9nsXJwDnA1cB51TVFUlOSrKqr/1KkubWsJMCzkpVnQecN23dn22j7WF91iJJmp2RDWZLkhYGg0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdeg2KJEckuTrJ+iRv28rzf5jkyiSXJflykv36rEeStP16C4oki4BTgBcDBwGrkxw0rdklwERVPQ34DPDevuqRJM1On2cUhwDrq+raqrobOBs4arBBVV1QVXe0ixcBy3qsR5I0C30GxVLghoHlDe26bTkO+MLWnkiyJslkksnNmzfNYYmSpJmMxWB2kmOACeB9W3u+qk6tqomqmli8eJ/5LU6SdnK79LjtjcDygeVl7bqfk+SFwJ8Cz6+qu3qsR5I0C32eUawDDkiyf5LdgKOBtYMNkhwMfARYVVU391iLJGmWeguKqroXOAE4H7gKOKeqrkhyUpJVbbP3AXsC5ya5NMnabWxOkjQifV56oqrOA86btu7PBh6/sM/9S5J+cWMxmC1JGl+9nlH04f774ZprRl2FJO08FlxQAFx44agrkKSFZq+Hz/aVCy4odtkFDj541FVI0kKzaNFsX+kYhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqVOvQZHkiCRXJ1mf5G1beX73JJ9un/9WkhV91iNJ2n69BUWSRcApwIuBg4DVSQ6a1uw4YFNVrQT+GnhPX/VIkmanzzOKQ4D1VXVtVd0NnA0cNa3NUcDH2sefAV6QJD3WJEnaTrv0uO2lwA0DyxuAZ2+rTVXdm2Qz8CjglsFGSdYAa9qleyYm9rmul4oXnLsWw+6bR13FeLAvtrAvtrAvtrht2Wxf2WdQzJmqOhU4FSDJZNWmiRGXNBaavrjDvsC+GGRfbGFfbJFkcrav7fPS00Zg+cDysnbdVtsk2QVYDPyox5okSdupz6BYBxyQZP8kuwFHA2untVkL/Fb7+BXAV6qqeqxJkrSderv01I45nACcDywCTq+qK5KcBExW1VrgNODMJOuBH9OEyUxO7avmBci+2MK+2MK+2MK+2GLWfRH/gJckdfGb2ZKkTgaFJKnT2AaF039sMURf/GGSK5NcluTLSfYbRZ3zYaa+GGj3m0kqyQ770chh+iLJq9rfjSuSfGq+a5wvQ/wbeXySC5Jc0v47OXIUdfYtyelJbk5y+TaeT5K/afvpsiTPHGrDVTV2PzSD398DngDsBnwHOGham98DPtw+Phr49KjrHmFf/BrwsPbxG3bmvmjb7QVcCFwETIy67hH+XhwAXALs0y7vO+q6R9gXpwJvaB8fBFw36rp76ovnAc8ELt/G80cCXwACHAp8a5jtjusZhdN/bDFjX1TVBVV1R7t4Ec13VnZEw/xeAPwFzbxhP5vP4ubZMH1xPHBKVW0CqKqb57nG+TJMXxTwiPbxYuDGeaxv3lTVhTSfIN2Wo4CPV+MiYO8kj51pu+MaFFub/mPpttpU1b3A1PQfO5ph+mLQcTR/MeyIZuyL9lR6eVV9fj4LG4Fhfi8OBA5M8o0kFyU5Yt6qm1/D9MWJwDFJNgDnAb8/P6WNne19PwEWyBQeGk6SY4AJ4PmjrmUUkjwEOBk4dsSljItdaC4/HUZzlnlhkqdW1a0jrWo0VgNnVNX7kzyH5vtbT6mq+0dd2EIwrmcUTv+xxTB9QZIXAn8KrKqqu+aptvk2U1/sBTwF+GqS62iuwa7dQQe0h/m92ACsrap7qur7wDU0wbGjGaYvjgPOAaiqbwJ7AEvmpbrxMtT7yXTjGhRO/7HFjH2R5GDgIzQhsaNeh4YZ+qKqNlfVkqpaUVUraMZrVlXVrCdDG2PD/Bv5HM3ZBEmW0FyKunY+i5wnw/TF9cALAJI8mSYofjivVY6HtcDr2k8/HQpsrqqbZnrRWF56qv6m/1hwhuyL9wF7Aue24/nXV9WqkRXdkyH7YqcwZF+cDxye5ErgPuCtVbXDnXUP2RdvAf4uyX+hGdg+dkf8wzLJWTR/HCxpx2PeAewKUFUfphmfORJYD9wBvH6o7e6AfSVJmkPjeulJkjQmDApJUieDQpLUyaCQJHUyKCRJnQwKaZok9yW5NMnlSf4xyd5zvP1jk3ywfXxikj+ay+1Lc82gkB7szqp6RlU9heY7Om8cdUHSKBkUUrdvMjBpWpK3JlnXzuX/5wPrX9eu+06SM9t1L23vlXJJki8lecwI6pd+YWP5zWxpHCRZRDPtw2nt8uE0cyUdQjOf/9okz6OZY+ztwH+oqluSPLLdxNeBQ6uqkvw28Mc03xCWFhSDQnqwhya5lOZM4irgi+36w9ufS9rlPWmC4+nAuVV1C0BVTd0PYBnw6Xa+/92A789P+dLc8tKT9GB3VtUzgP1ozhymxigCvLsdv3hGVa2sqtM6tvM/gQ9W1VOB36GZiE5acAwKaRvauwb+AfCWdir784H/nGRPgCRLk+wLfAV4ZZJHteunLj0tZssUzr+FtEB56UnqUFWXJLkMWF1VZ7ZTVH+znaX3duCYdqbSdwJfS3IfzaWpY2nuqnZukk00YbL/KI5B+kU5e6wkqZOXniRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTp/wNbDPb9b4LiqQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluate(dummy, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameter tuning with GridsearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 108 candidates, totalling 324 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers.\n",
      "[Parallel(n_jobs=10)]: Done  30 tasks      | elapsed:   30.6s\n",
      "[Parallel(n_jobs=10)]: Done 180 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=10)]: Done 324 out of 324 | elapsed:  2.9min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best options\n",
      "=======================================\n",
      "criterion: 'entropy'\n",
      "max_depth: None\n",
      "max_features: 'auto'\n",
      "n_estimators: 500\n"
     ]
    }
   ],
   "source": [
    "param_grid=param_grid = { \n",
    "    'n_estimators': [200, 500, 1000],\n",
    "    'max_features': ['auto', 'sqrt', 'log2'],\n",
    "    'max_depth' : [None, 4,5,6,7,8],\n",
    "    'criterion' :['gini', 'entropy']\n",
    "}\n",
    "best_parameters = hyper_parameter_tuning(param_grid , RandomForestClassifier(random_state=0), 3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training with best parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XmcFNW5//HPM/vKsClhkRmuQWVfxQVR+KFeTIxGJXHBGLOI4Wq8bomoiTHcEI3JJSaRqHijicko0ajRxC2GQNQYFVAWBRfUQVkEZZmZnr27z++Pri66Z+2B6Z4Z5vvWfnXX6VNVT9c056k6VXXanHOIiIgApHV2ACIi0nUoKYiIiE9JQUREfEoKIiLiU1IQERGfkoKIiPiUFERExKekICIivqQlBTO718x2mtkbLbxvZvZLM9tkZuvMbGKyYhERkcRkJHHZvwXuAO5v4f3TgOHe4xjgTu+5Vf3793clJSXti2T3bti6FerrISsLBg+Gvn3bnK2qqor8/Pz2rSsJcbRXm3GnKI42NYqjtl8/cgYN6tQY2rstOuw7kuK/SYtxd9HvRjSODv83uZ9xtEeHxNwBcaxevfpT59whbVZ0ziXtAZQAb7Tw3t3A+THTbwMD21rmpEmTXLv84Q/O5eU5B/seeXmR8jYsX768fetKUhzt1WrcKYyjVc3EEczOTm0cHbAtOuQ70gl/k2bj7sLfjWgcHfpv8gDiaI8DjrmD4gBWuQTabXNJHPvIzEqAvzrnRjfz3l+BW51zL3rTy4DrnHOrWlvm5MmT3apVrVaJV1ICmzc3Lc/OhmOPbXXWvXv30rt378TX1ZqXX4a6uv2Ko71ajTuFcbSqK8TRATF0yHekE7ZFs3F3hb9JG3HsPeqojvs3eQBxtGd7HPB3pKU4iouhrCzhxZjZaufc5LbqJbP7qMOY2VxgLsCAAQNYsWJFwvOe9OGHWDPlrq6O8j17Wp03FA6zt406iSqqq9vvONqrtbhTGUdrukIcHRFDR3xHOmNbNBd3V/ibtBVHKBTqsH+TBxJHe7bHgX5HWozjww/5ZzvawoQlcjixvw+6QvdRcXH8YVf0UVzc5qwdeqh6AHG0V6txpzCOVnWFODoghg75jnTCtmg27q7wN2kjjpR2H3XQ9jjgmDsoDhLsPurMS1KfAC7yrkI6Fih3zm3v8LUsXAh5efFleXmR8lRSHG3GEcrOTm0cXXhb9PQ4SidlUnIlpP0ASq6E0kmZiiNFcSTzktQHgX8DR5rZFjP7hpl9y8y+5VV5Cngf2ATcA/xXUgKZMweWLIn0v5lFnpcsiZSnkuJoM463r702tXF04W3Rk+MoHQtzzzA29wZnsLl3ZLp0bErDSHoc0T3zsAsTCocIhoM0hBqoD9VTF6yjNlhLTUMN942sT+n2SOqJ5mRo94nmA7BixQqmT5+eknV1JMWdOt0xZmg57tL1pdy47EY+LP+QoUVDWThzIXPGdExScM5RG6wlUB+gsr4y8lxX6U9X1lVSWV/JTctvoryuvMn8BZkFzOg/g88M/AyttVuOVt5rYb7m5nnozYeoaqhqUp6Xmcfpw08n7MKECRMOhwm5EM45Qi4UaeRdyC+vLK8ktzA3Uj/6nvc6mhQc+xJEbHmYMJv3bibkQk3iKC4qpuzKshY/a2MH1YlmkYNd6fpSblh2Ax+Vf8RhRYfxoxk/4oIxF2AWf4rRGp1ybPz+gcYw9y9zqW6oBmBz+WYueeISdlfv5pTDT4k02nWVVNRXEKgLUFFXQWVD5b6Gva6SQEOAQF0gruEP1Aeoaqiiqr6q2cYtUYGGAC98+gJpeyMdHI23RazWtktL8zWep7mEAFDdUM2/t/ybNEvzH4ZhZnFl0Ud9qB4LRt5Lt3Sy0rL2zefNEzcv8e+9v+f9ZuP4sPzDFj/jgVBSEEmi2L3HaDdB2IUJhoMEw0HqQ/X8acOfuH7Z9dQEa4DIP/a5f53LzqqdfOGIL+xbWOO2LGbnNuzC1IZqqW6opqq+iuqGamoaavzGuLqhmupgNdUN1QTqA2zbto07dt5BVX0VVQ2R99fuWEswHIxbRU2whiueuaLNz5mbkUtuZi75mflxz/3z+pOXmUd+Zj75Wfn+64Ksgn3PWfn0yu5FYVYhhdmFfO6Bz7GtcluTdQzpNYTfjv8tE49t3+AH0Qa7vcbcOYaPKj5qNo7XL32dsAsDxD3H7vFHy95e/TbDJw4H9h2phAnvu+gmGlr079loevX21c1uj6FFQ9v9mRKhpCA92v50l0Qb9uhhfqA+4PcJRxv6YDjoJ4PYxtvhMDNC4RBV9VXUBGv48Ys/9hNCVG2wlgXPL6CsvIzq+mqqGqoI1AcijX70tVcebdQTlZWeRbZl0yvQy2+o8zLymiSEWD875WfkZ+Xva9wz8ijKLvIb9Mz0TNIsjYy0jLhnf08YazIdfd3YbafcFnfEApEum1tPvpX0Xen0ye2T8Gc9ELecfEuLcfTL65fwcj5I/4BhfYa1+H40UUS7sBpP3zLzFuY9Oa9JHAtnJudEs5KC9BiN/7GVrivlW09+K667ZO4Tc6kP1jN75Gy/kW8IN1AfrKemoYbyunKq6qsINES6RN779D1eX/16ZE/c20uPNtJ+vfoqKusr457rQs3cjNRIRV0Fd626i4LMAvKy9u1h52XmMaTXEL+Bzs/c11jHTudlRRr7nIwccjNzycnIISc9h8z0TMrWljFs3DAwSLM0MtMymXrvVLZWbm0Sx9BeQ7nimCuaNOjJEk3KzSXr9tyjlMw4OlJ0W/rdWo027UXjLiI9LT3pcUQpKUjKxJ5IC7swD6x/gB+s+AFbKrYwpNcQLhx4IeNqxgHNH5I75/D/i5lufBgfe4gO0BBsIFAfoKKhwu8Xr6yr5MblNzbZw64OVjPvyXnc8eod/h55tAumIdzQ/AfbED+Zl5lHQVaB/8jPzGdIryEUZBZQkF1AQWaky6Qwq5CCrAJ+9MKP2F2zu8liBxUOYuUlK1vcliEXij9qwRH5P3I0YhjpaelkpmWSlZ5FVnoWGWkZpKelsy19G//R9z9It3S/UfrJKT9pds/4xyf/mMz0zBb/rskwZ8ycpDV6iqN1Sgqy3/zL6WL6zBtfXhfd0/a7Ujx/efsvfG/596gN1gLwUcVHLAosov+a/pxx5BlxJwMbQg0EGiInNgP13gnOusrIc33kipWKuoomj9g67VEXqiM3M5dD8w8lPyuyd16YVeg35NGygqwCdm7aydjxY+MSQHpaervWl5meyXef+25cF1JuRi7XHn8t1Q3VhMPedjP8Rj/a2Odk5JCRlhHX4EdPaEa7a5pjGBlp8f/8U7VnLF2bkoL4Yhv22EcwtK8bJdqlEtvIR/fMow154ysqMtIySHfpVNRXsLtmN3tq9nDLi7f4CSGqLlzHDctu4ME3HqSyrpLyunIq6ira7C83jF7ZveIexUXF/uui7CJ65fSiV9a+94tyirjosYvYUbWjyfIGFQ7i3jPvjUw0OvnnnMMwf2/8gx0fMKhgkF+nJljjzxOtE50ndjmxyz318FOpDdby83//nO2B7QwsHMh1U6/j3FHnxjX26ZbuN/rN9cV3hK6yZyydR0mhh4he8RIKh3hg/QPc/M+b2VqxlUGFg7jm+Gv4whFfIBQONXu5XuNL7QCq6qvYW7s30sjX7vEb+901u9ldu+919L3y2vJWrx+PqgvVkZORw6F5h0Ya8JyYhr1Rwx8ty8/Kj2sk464Xj+leMbO47pXrpl7Hjf+4scke+g9P+iEDCwb6nx3iL2OMLduatpWhvYfG1YndO29c1tL0Ncddw7XHX9vm9hFJNiWFg0S0AYw2hntq9lAfqqc2WEtDuIFwONLn/Nd3/hrXbbO1cis3LLuBTwKfMGnQpPhGvlFjv6d2D3tq9jR7Y1FUTnoOfXL70Ce3D31z+zK412D65ERe+8+5fbjqmavYWb2zyfyDCwfzx9l/bHH5sZ8x7CI3DlXXV4Pt24tPS0sjwzLITM8kPzOfrPSsJnva6ZbOVcddxaEFhx5Qd4mZkZORk3B9ka5OSaGbiD2xGHvpY22wNtJ374J+o9gQbmBXzS7SLI2GUAPbK7ezuWIzZXvL+Om/ftqk26Y2WMst/7qlyTrzMvPiGvLiomK/sY8tj53OzcxN6PN8/6TvN+lHz07L5trjr6WmoWbfTU6OuAY/Mz2TjLQMcjNym3StpFma3+gnSt0lIvGUFLqQ6F5wtJunLlhHXajOv+492gXinPO7ctLT0qkJ1vBRxUeU7S2jbG8ZazatoaKsgrK9ZXwc+DihbhuAP87+Y6SRz4k09vuzB9z4CqPYm3liLwk99fBTqQvWsejlRWyvjPSjf2XgV+L60RufOG3vCVwRaT8lhRRyzsWdqI0OfFUfqo908Xg3OkX7vP29X0tnT/0eNu/dTFl5pOHfvHezP723dm/cevpk9uHwQw7n+MOOZ1jvYRT3Lqa4qJiS3iWcVnpas9eiDy4czAlDT/CnG/fHRxv4xjdj+Z/Nizkab/SKmOiJ5tgGPnoj0xXHXMGVx17pn5ResWIFAwsHdvh2F5HEKSl0sGg3T2wXT3TEw0c3Psqify+KXGFSMJBrjr+GLx71xUijaRlsDWxlc3mkm8d/3ruZzeWb47p80iyNIb2GUFxUzOlHnE5JUQklvUv8xr9sbRmjjh7VbGzfnfpd5v99fly3TU5GDlcdexWBusC+daTFN+7Rhj22i6bx2C+tXQIpIt2DksJ+iu3bbwg1UBus9bt6ojdVxe45/+Wdv/D95d/3G+NtgW1897nvsmT1EmqCNWyt2Bo3WFhOek6kke9dzInFJ0YafW9vf0ivIa3fTOSgLljnn5CN9sebGWcccQbpls6t/7qVrRVbGdJrCAtmLOCCMRc0GdxLRHoeJYU2RPvzG0IN1IUie/z1ofq4a/Sje88ZaRnkZeY126De8sItTca3CbkQ7+95n88N/xxfPOqLlBTt29sfUDCg1ROm0aQUHaI3yixyDX1ORg45GTn+idmMtAz/7tXLplzGZVMu66AtJCIHkx6fFGL7+YPhoN/VE+32KdtTFtfHH73yJdE96bc+fYvFKxfzcdXHzb4fDAf59ed/3ex70RPPoXDkET1hbGZkpmWSnZFNdno22RnZcQ3/1vSt6psXkf3So5JCdG8/elVPbbA2fmRIR+TyxrR0sjKySEtLoyC7YL/WtWrbKu549Q6ee/85fyycQH2gSb1BhYOoD9X7DX/svWMZlkF2RjaFWYVkZ2T7SSkjLUPdOyKSFD0qKeyu2c2u6l1kZWSRbvvGjukozjlWlK3gjlfv4OWtL9M7pzfXHHcNF4+/mBVlK5pcl5+TkcM1x11DZlomBZkF8Xv87bzeXkSkI/SopACQk5nT4XeghsIh/vruX1n86mLe/ORNBhYM5ObpNzNnzBzyMiM/hH72iLMJhUPc9q/b2B7YzmFFh7Hw/y3kwrEXdmgsIiIHosclhY5UF6zjTxv+xK9X/ZqyvWUc3udwFp26iLNGnEVWelZc3ZqGGj4//PPMnTQ34bt+RURSTUlhPwTqA/xh3R9YsnoJO6p2MHbAWJacvoRZn53V7F23gboA2RnZDCoclPJx6UVE2kNJoR12Ve/i3tfv5bdrfsveur1MPWwqt8+6nWlDpzV74jfswgTqA/TJ6cMh+YfoHIGIdHlKCgnYWrGVu1ffTen6UmqDtZz22dO47OjLmDBwQovzNIQaqAnWMLBgIEU5RSmMVkRk/ykptOLD6g/5v2f/j0c3PgrAWUedxWVHX8bwfsNbna+mIXKFUXFRsc4fiEi3oqTQjDUfr+GOV+/gmU3PkJ2RzVfHfZVLJ13K4F6DW53POUdVQxW5GbkMLBzY5OcORUS6OrVaHuccL3z4AotXLubFD1+kKLuICw67gOs+fx398vq1OX/YhQnUBeiX149+ef10/kBEuqUenxTCLswzm57hjlfvYO2OtQzIH8D3T/w+F469kM1rNyeUEKJDYgwqHESvnF4piFpEJDl6RFIoXV/q/+TiwMKBXH/C9Zx+xOk8tvExFq9czHt73qOkdwm3nXwbs0fOJjsjO+Fl++cPehfrZxlFpNs76JNC6fpS5v5lLtUN1QBsq9zG1c9ezff+8T3K68oZdcgo7vz8nXx++Ofb9ctezjkC9QHyMvN0/kBEDhoHfUt247Ib/YQQ1RCOXC76h7P+wPSS6e0eXC4UDlFVX0W/vH70z+uvwelE5KBx0CeFD8s/bLa8IdTAjGEz2r28+lA99aF6nT8QkYPSQX+JzNCioc2WDyoc1O5lVTdUEw6HKS4qVkIQkYPSQZ8UFs5c6I9UGpWbkcv8E+YnvAznHJV1leRk5DC099B2nYgWEelODvruozlj5gA0ufro7BFnJzR/KByiqqGK/rn96ZfXT+cPROSgdtAnBYgkhjlj5rAjsIOqhqqELx11zlHdUM3gwsEUZhcmOUoRkc7XI5LC/qiuj1yxVNK7RN1FItJjHPTnFNorev4gNzOXrPQsJQQR6VGUFGKEwiEq6yvpn9d/v65OEhHp7tR95KkL1tEQamBI4RAKsgs6OxwRkU6R1CMFM5tlZm+b2SYza3INqJkNNbPlZva6ma0zs88lM56WVNdX43CU9ClRQhCRHi1pScHM0oHFwGnASOB8MxvZqNr3gIeccxOA84BfJyue5kTPH+Rl5lFcVExWelYqVy8i0uUk80hhCrDJOfe+c64eWAqc2aiOA6K3BhcB25IYT5xgOEhlXSWH5B/CwMKB7RoMT0TkYJXMcwqDgY9iprcAxzSqczPwNzP7NpAPnJzEeHy1wVqCoSCHFR1GflZ+KlYpItItmHMuOQs2mw3Mcs5905v+CnCMc+7ymDpXezH8r5kdB/wGGO2cCzda1lxgLsCAAQMmLV26dL9iCoaDBMNB0iyNzPRMjNbvTg4EAhQUdL9zDIo7dbpjzKC4U6mrxDxjxozVzrnJbVZ0ziXlARwHPBszfT1wfaM6bwKHxUy/Dxza2nInTZrk9tfOwE63rWKbC4aCCdVfvnz5fq+rMynu1OmOMTunuFOpq8QMrHIJtN3JPKewEhhuZsPMLIvIieQnGtX5EJgJYGYjgBzgk2QF1C+vH58p+IzOH4iItCBpScE5FwQuB54FNhK5yuhNM1tgZmd41a4BLjGztcCDwMVeRkuKNEvTgHYiIq1I6s1rzrmngKcald0U83oDMDWZMYiISOI0zIWIiPiUFERExKekICIiPiUFERHxKSmIiIhPSUFERHxKCiIi4lNSEBERn5KCiIj4lBRERMSnpCAiIj4lBRER8SkpiIiIT0lBRER8SgoiIuJTUhAREZ+SgoiI+JQURETEp6QgIiI+JQUREfEpKYiIiE9JQUREfEoKIiLiU1IQERGfkoKIiPiUFERExKekICIiPiUFERHxKSmIiIhPSUFERHxKCiIi4lNSEBERn5KCiIj4lBRERMSnpCAiIj4lBRER8SkpiIiIT0lBRER8SgoiIuJLalIws1lm9raZbTKz+S3U+bKZbTCzN83sgWTGIyIirctI1oLNLB1YDJwCbAFWmtkTzrkNMXWGA9cDU51ze8zs0GTFIyIibUv4SMHMTjCzr3mvDzGzYW3MMgXY5Jx73zlXDywFzmxU5xJgsXNuD4BzbmfioYuISEcz51zblcx+AEwGjnTOHWFmg4CHnXNTW5lnNjDLOfdNb/orwDHOuctj6vwZeAeYCqQDNzvnnmlmWXOBuQADBgyYtHTp0nZ8xP0XCAQoKChIybo6kuJOne4YMyjuVOoqMc+YMWO1c25yW/US7T46C5gAvAbgnNtmZoUHEF/s+ocD04EhwPNmNsY5tze2knNuCbAEYPLkyW769OkdsOq2rVixglStqyMp7tTpjjGD4k6l7hZzot1H9S5ySOEAzCw/gXm2AofFTA/xymJtAZ5wzjU45z4gctQwPMGYRESkgyWaFB4ys7uB3mZ2CfB34J425lkJDDezYWaWBZwHPNGozp+JHCVgZv2BI4D3E4xJREQ6WELdR865n5nZKUAFcCRwk3PuuTbmCZrZ5cCzRM4X3Ouce9PMFgCrnHNPeO+damYbgBDwHefcrgP4PCIicgDaTArepaV/d87NAFpNBI05554CnmpUdlPMawdc7T1ERKSTtdl95JwLAWEzK0pBPCIi0okSvfooAKw3s+eAqmihc+6KpEQlIiKdItGk8Kj3EBGRg1iiJ5p/511BdIRX9LZzriF5YYmISGdIKCmY2XTgd0AZYMBhZvZV59zzyQtNRERSLdHuo/8FTnXOvQ1gZkcADwKTkhWYiIikXqI3r2VGEwKAc+4dIDM5IYmISGdJ9EhhlZn9H/AHb3oOsCo5IYmISGdJNCnMAy4DopegvgD8OikRiYhIp0k0KWQAv3DOLQL/LufspEUlIiKdItFzCsuA3JjpXCKD4omIyEEk0aSQ45wLRCe813nJCUlERDpLokmhyswmRifMbDJQk5yQRESksyR6TuFK4GEz2+ZNDwTOTU5IIiLSWVo9UjCzo83sM865lcBRwB+BBuAZ4IMUxCciIinUVvfR3UC99/o44AZgMbAH7zeTRUTk4NFW91G6c2639/pcYIlz7hHgETNbk9zQREQk1do6Ukg3s2jimAn8I+a9RM9HiIhIN9FWw/4g8E8z+5TI1UYvAJjZZ4HyJMcmIiIp1mpScM4tNLNlRK42+pv3m8oQOcL4drKDExGR1GqzC8g593IzZe8kJxwREelMid68JiIiPYCSgoiI+JQURETEp6QgIiI+JQUREfEpKYiIiE9JQUREfEoKIiLiU1IQERGfkoKIiPiUFERExKekICIiPiUFERHxKSmIiIhPSUFERHxKCiIi4lNSEBERX1KTgpnNMrO3zWyTmc1vpd45ZubMbHIy4xERkdYlLSmYWTqwGDgNGAmcb2Yjm6lXCPw38EqyYhERkcQk80hhCrDJOfe+c64eWAqc2Uy9/wF+AtQmMRYREUlAMpPCYOCjmOktXpnPzCYChznnnkxiHCIikqCMzlqxmaUBi4CLE6g7F5gLMGDAAFasWJHU2KICgUDK1tWRFHfqdMeYQXGnUreL2TmXlAdwHPBszPT1wPUx00XAp0CZ96gFtgGTW1vupEmTXKosX748ZevqSIo7dbpjzM4p7lTqKjEDq1wCbXcyu49WAsPNbJiZZQHnAU/EJKNy51x/51yJc64EeBk4wzm3KokxiYhIK5KWFJxzQeBy4FlgI/CQc+5NM1tgZmcka70iIrL/knpOwTn3FPBUo7KbWqg7PZmxiIhI23RHs4iI+JQURETEp6QgIiI+JQUREfEpKYiIiE9JQUREfEoKIiLiU1IQERGfkoKIiPiUFERExKekICIiPiUFERHxKSmIiIhPSUFERHxKCiIi4lNSEBERn5KCiIj4lBRERMSnpCAiIj4lBRER8SkpiIiIT0lBRER8SgoiIuJTUhAREZ+SgoiI+JQURETEp6QgIiI+JQUREfEpKYiIiC+jswPoCA0NDWzZsoXa2toOXW5RUREbN27s0GWmQnePOycnhyFDhpCZmdnZIYn0OAdFUtiyZQuFhYWUlJRgZh223MrKSgoLCztseanSneMuKChg165dbNmyhWHDhnV2SCI9zkHRfVRbW0u/fv06NCFI5zAz+vXr1+FHfSKSmIMiKQBKCAcR/S1FOs9BkxQ6065duxg/fjzjx4/nM5/5DIMHD/an6+vrE1rG1772Nd5+++1W6yxevJjS0tKOCFlEpFkHxTmFdisthRtvhA8/hKFDYeFCmDNnvxfXr18/1qxZA8DNN99MQUEB1157bVwd5xzOOdLSms/D9913X5vrueyyy/Y7xmRq67OJSPfR8/4Vl5bC3LmweTM4F3meOzdS3sE2bdrEyJEjmTNnDqNGjWL79u3MnTuXyZMnM2rUKBYsWODXPeGEE1izZg3BYJDevXszf/58xo0bx3HHHcfOnTsB+N73vsftt9/u158/fz5TpkzhyCOP5KWXXgKgqqqKCy+8kJEjRzJ79mwmT57sJ6xY3/nOdxg5ciRjx47luuuuA+Djjz/mzDPPZOzYsYwbN45XXnkFgNtuu43Ro0czevRofvWrX7X42Z5++mmOO+44Jk6cyLnnnktVVVWHb1MRSa6D70jhyiuhmUbQ9/LLUFcXX1ZdDd/4BtxzT1xxbigE6ekwfjx4jXF7vfXWW9x///1MnjwZgFtvvZW+ffsSDAaZMWMGs2fPZuTIkXHzlJeXc9JJJ3Hrrbdy9dVXc++99zJ//vwmy3bO8eqrr/LEE0+wYMECnnnmGX71q19x6KGH8vjjj7N27VomTpzYZL4dO3bw1FNP8eabb2Jm7N27F4gciZxyyilcfvnlBINBqqureeWVVygtLWXlypUEg0GmTJnC9OnTyc3NjftsO3fu5NZbb2XZsmXk5eWxcOFCfvGLX3DDDTfs13YTkc7R844UGieEtsoP0OGHH+4nBIAHH3yQiRMnMnHiRDZu3MiGDRuazJObm8tpp50GwKRJkygrK2t22WeffXaTOi+++CKzZ88GYNy4cYwaNarJfH379iUtLY1LLrmExx57jPz8fABWrFjBpZdeCkBGRga9evXixRdf5JxzziE3N5fCwkK++MUv8sILLzT5bC+99BIbNmzg+OOPZ/z48ZSWlrYYt4h0XQffkUJbe/QlJZEuo8aKi2HFiriimg643j/a4AK8++67/OIXv+DVV1+ld+/eXHjhhc1eepmVleW/Tk9PJxgMNrvs7OzsNus0JzMzk1WrVvHcc8/x8MMPc+edd/K3v/0NaN+VP7GfzTnHrFmz+P3vf5/w/CLS9ST1SMHMZpnZ22a2ycya9H+Y2dVmtsHM1pnZMjMrTmY8QOSkcl5efFleXqQ8ySoqKigsLKRXr15s376dZ599tsPXMXXqVB599FEA1q9f3+yRSGVlJRUVFZx++un8/Oc/5/XXXwdgxowZ3HXXXQCEQiEqKiqYNm0ajz32GDU1NQQCAR5//HGmTZvWZJnHH388//znP3n//feByLmNd999t8M/n4gkV9KOFMwsHVgMnAJsAVaa2RPOudhW6nVgsnOu2szmAbcB5yYrJmDfVUYdePVRoiYk2pkgAAARPUlEQVROnMjIkSM56qijKC4uZurUqR2+jm9/+9tccMEFjBw50n8UFRXF1SkvL+fss8+mrq6OcDjMokWLALjjjju45JJLuPvuu8nIyODuu+9mypQpnH/++Rx99NEAzJs3jzFjxrBp06a4ZQ4YMIDf/OY3nHvuuf5luD/+8Y8ZPnx4h39GEUmi6OWEHf0AjgOejZm+Hri+lfoTgH+1tdxJkya5xjZs2NCkrCNUVFQkZbnJ1NDQ4Hbu3Omcc+6dd95xJSUlrqGhoZOjSkzs9k7W37SjLV++vLND2C+KO3W6SszAKpdA253McwqDgY9iprcAx7RS/xvA00mMp0cIBAKceuqphMNhnHP+Xr+ISCK6RGthZhcCk4GTWnh/LjAXIt0UKxqdEC4qKqKysrLD4wqFQklZbjKlp6ezfPly0tPT/bLu8hlit3dtbW2Tv3NXFAgEukWcjSnu1OluMSczKWwFDouZHuKVxTGzk4EbgZOcc81eF+qcWwIsAZg8ebKbPn163PsbN25Myqig3Xm00e4ed05ODhMmTOjkiNq2YsUKGn8fuwPFnTrdLeZkXn20EhhuZsPMLAs4D3gitoKZTQDuBs5wzu1MYiwiIpKApCUF51wQuBx4FtgIPOSce9PMFpjZGV61nwIFwMNmtsbMnmhhcSIikgJJPafgnHsKeKpR2U0xr09O5vpFRKR9et4wF0kwY8aMJjei3X777cybN6/V+QoKCgDYtm2bPzRFY9OnT2fVqlWtLuf222+nurranz7nnHP88YxERNqjRyaF0vWllNxeQtoP0yi5vYTS9Qc2Qur555/P0qVL48qWLl3K+eefn9D8gwYN4k9/+tN+r79xUnjkkUfo3bv3fi8vWUKhUGeHICJt6HFJoXR9KXP/MpfN5ZtxODaXb2buX+YeUGKYPXs2Tz75pH8nb1lZGdu2bWPatGkEAgFmzpzJxIkTGTNmDI8//niT+cvKyhg9ejQANTU1nHfeeYwYMYKzzjqLmpoav968efP8Ybd/8IMfAPDLX/6Sbdu2MWPGDGbMmAHA6NGj+fTTTwFYtGiRP+x1dNjtsrIyRowYwSWXXMKoUaM49dRT49YT9fDDDzN69GjGjRvHiSeeCEQa9muvvZbRo0czduxYfyjtZcuWMWHCBMaMGcPXv/516rwBBktKSrjuuuuYOHEiDz/8MO+99x6zZs1i0qRJTJs2jbfeemu/t7uIdLwucZ9CR7rymStZ83HLQ2e/vOVl6kLxV75WN1Tzjce/wT2r44fODoVCpKenM/4z47l9VssD7fXt25cpU6bw9NNPc+aZZ7J06VK+/OUvY2bk5OTw2GOP0atXLz799FOOPfZYzjjjjBYHnrvzzjvJy8tj48aNrFu3Lm7o64ULF9K3b19CoRAzZ85k3bp1XHHFFSxatIjly5fTv3//uGWtXr2a++67j1deeQXnHMcccwwnnXQSffr04d133+XBBx/knnvu4ctf/jKPPPIIF154Ydz8CxYs4Nlnn2Xw4MF+d9SSJUsoKytjzZo1ZGRksHv3bmpra7n44otZtmwZRxxxBBdddBF33nknV155JRD5EaLXXnsNgJkzZ3LXXXcxfPhwXnnlFf7rv/6Lf/zjHy1uWxFJrR53pNA4IbRVnqjYLqTYriPnHDfccANjx47l5JNPZuvWrezYsaPF5Tz//PN+4zx27FjGjh3rv/fQQw8xceJEJkyYwJtvvtnsYHexXnzxRc466yzy8/MpKCjg7LPP9oe9HjZsGOPHjwdaHp576tSpXHzxxdxzzz1+18/f//53Lr30Uv8u6b59+/L2228zbNgwjjjiCAC++tWv8vzzz/vLOffcyHBWgUCAl156iS996UuMHz+eSy+9lO3bt7f6GUQktQ66I4XW9ugBSm4vYXN506Gzi4uKWXHxiriy9twEduaZZ3LVVVfx2muvUV1dzaRJkwAoLS3lk08+YfXq1WRmZlJSUtLscNlt+eCDD/jZz37GypUr6dOnDxdffPF+LScqOuw2RO6Cbq776K677uKVV17hySefZNKkSaxevXq/1hUdYjscDtO7d+9mfwlORLqGHneksHDmQvIy44fOzsvMY+HMAxs6u6CggBkzZvD1r3897gRzeXk5hx56KJmZmSxfvpzNzf2WQ4wTTzyRBx54AIA33niDdevWAZFht/Pz8ykqKmLHjh08/fS+YaIKCwubHcpi2rRp/PnPf6a6upqqqioee+yxZoe9bsl7773HMcccw4IFCzjkkEP46KOPOOWUU7j77rv932/YvXs3Rx55JGVlZf7Iqb///e856aSmI5b06tWLYcOG8fDDDwORo6i1a9cmHI+IJF+PSwpzxsxhyReWUFxUjGEUFxWz5AtLmDPmwIfOPv/881m7dm1cUpgzZw6rVq1izJgx3H///Rx11FGtLmPevHkEAgFGjBjBTTfd5B9xjBs3jgkTJnDUUUdxwQUXxA27PXfuXGbNmuWfaI6aOHEiF198MVOmTOGYY47hm9/8ZruGjvjOd77DmDFjGD16NMcffzzjxo3jm9/8JkOHDvV/x/mBBx4gJyeH++67jy996UuMGTOGtLQ0vvWtbzW7zNLSUn7zm9/4vwrX3Il3Eek8FhlRtfuYPHmya3zd/saNGxkxYkSHr+tgGEOoO4mNO1l/047W3ca1iVLcqdNVYjaz1c65yW3V63FHCiIi0jIlBRER8SkpiIiI76BJCt3t3Ii0TH9Lkc5zUCSFnJwcdu3apcbkIOCcY9euXeTk5HR2KCI90kFx89qQIUPYsmULn3zySYcut7a2tls2Tt097pycHIYMGdLZ4Yj0SAdFUsjMzGTYsGEdvtwVK1Z0i5+EbExxi8j+Oii6j0REpGMoKYiIiE9JQUREfN1umAsz+wRofVS5jtMf+DRF6+pIijt1umPMoLhTqavEXOycO6StSt0uKaSSma1KZKyQrkZxp053jBkUdyp1t5jVfSQiIj4lBRER8SkptG5JZwewnxR36nTHmEFxp1K3ilnnFERExKcjBRER8fXYpGBmh5nZcjPbYGZvmtl/e+U3m9lWM1vjPT4XM8/1ZrbJzN42s//sxNjLzGy9F98qr6yvmT1nZu96z328cjOzX3pxrzOziZ0U85Ex23SNmVWY2ZVdcXub2b1mttPM3ogpa/f2NbOvevXfNbOvdkLMPzWzt7y4HjOz3l55iZnVxGzzu2LmmeR9tzZ5n8s6Ie52fyfMbJZXtsnM5icz5lbi/mNMzGVmtsYr7zLbOyHOuR75AAYCE73XhcA7wEjgZuDaZuqPBNYC2cAw4D0gvZNiLwP6Nyq7DZjvvZ4P/MR7/TngacCAY4FXusC2Twc+Boq74vYGTgQmAm/s7/YF+gLve899vNd9UhzzqUCG9/onMTGXxNZrtJxXvc9h3uc6rRO2dbu+E97jPeA/gCyvzshUx93o/f8Fbupq2zuRR489UnDObXfOvea9rgQ2AoNbmeVMYKlzrs459wGwCZiS/EgTdibwO+/174AvxpTf7yJeBnqb2cDOCDDGTOA951xrNyF22vZ2zj0P7G4mnvZs3/8EnnPO7XbO7QGeA2alMmbn3N+cc0Fv8mWg1aFnvbh7OededpEW6372fc6kaGFbt6Sl78QUYJNz7n3nXD2w1KubNK3F7e3tfxl4sLVldMb2TkSPTQqxzKwEmAC84hVd7h1y3xvtJiCSMD6KmW0LrSeRZHLA38xstZnN9coGOOe2e68/BgZ4r7tS3FHnEf8Ppqtvb2j/9u1q8X+dyJ5o1DAze93M/mlm07yywUTijOrMmNvznehq23oasMM5925MWVff3r4enxTMrAB4BLjSOVcB3AkcDowHthM5DOxqTnDOTQROAy4zsxNj3/T2OrrkZWVmlgWcATzsFXWH7R2nK2/f5pjZjUAQKPWKtgNDnXMTgKuBB8ysV2fF14xu951o5Hzid3q6+vaO06OTgpllEkkIpc65RwGcczuccyHnXBi4h31dFluBw2JmH+KVpZxzbqv3vBN4jEiMO6LdQt7zTq96l4nbcxrwmnNuB3SP7e1p7/btEvGb2cXA6cAcL5nhdb/s8l6vJtIff4QXX2wXU6fEvB/fiS6xrQHMLAM4G/hjtKyrb+/GemxS8Pr9fgNsdM4tiimP7W8/C4heXfAEcJ6ZZZvZMGA4kZNEKWVm+WZWGH1N5GTiG1580Stcvgo87r1+ArjIu0rmWKA8phukM8TtRXX17R2jvdv3WeBUM+vjdX+c6pWljJnNAr4LnOGcq44pP8TM0r3X/0Fk277vxV1hZsd6/z4uYt/nTGXc7f1OrASGm9kw70j0PK9uZzgZeMs553cLdfXt3URnn+nurAdwApEugHXAGu/xOeD3wHqv/AlgYMw8NxLJ8m/TSVcJELnCYq33eBO40SvvBywD3gX+DvT1yg1Y7MW9Hpjcids8H9gFFMWUdbntTSRpbQcaiPTzfmN/ti+RfvxN3uNrnRDzJiJ97dHv911e3XO8784a4DXgCzHLmUykEX4PuAPvBtcUx93u74T3b/cd770bO+M74pX/FvhWo7pdZnsn8tAdzSIi4uux3UciItKUkoKIiPiUFERExKekICIiPiUFERHxKSlIl2Nm/WJGlPy40YiZWQku4z4zO7KNOpeZ2ZyOibprMLMXzWx8Z8ch3ZcuSZUuzcxuBgLOuZ81Kjci399wpwTWRZnZi8Dlzrk1nR2LdE86UpBuw8w+a5HfvyglcjPQQDNbYmarLPKbGDfF1H3RzMabWYaZ7TWzW81srZn928wO9er8yMyujKl/q5m9apFx+Y/3yvPN7BFvvX/y1tVkT9zMjvYGO1ttZk+b2QAzy/SmT/Dq/NTMfui9/qGZrTSzN8zsLi/JReNY5K1ng5lNtshvIbzrJcjodnjTzJaa2UYze8jMcpuJ6TTv875mkbH+82Pi2GCRAed+0qF/JOn2lBSkuzkK+LlzbqSLjAE13zk3GRgHnGJmI5uZpwj4p3NuHPBvIncaN8ecc1OA7wDRBPNt4GPn3Ejgf4iMphs/k1k28AvgHOfcJOAPwP845xqArwFLzOxUYAbwI2+2XzjnjgbGePHFDqtd432m3wB/Br7l1Ztr3g/lEPltgdudcyOAWuDSRjEdSuR3H2a6yOCJ64D/NrMBRO7+HeWcGwvc0sK2kB5KSUG6m/ecc6tips83s9eIDB8wgkhj2ViNcy46bPRqIj960pxHm6lzApHx+XHORYcWaWwEMAr4u0V+bWs+3gBtzrl13vyPA1/3EgXATDN7lchwJSd580dFx+1ZD6x3kQHiaon8uFJ0ALUPXOT3GyCShE5oFNPxRLbFS15Mc7zPtBsIA/eY2VlAVQvbQnqojM4OQKSd/EbMzIYD/w1Mcc7tNbM/ADnNzFMf8zpEy9/7ugTqNMeAdc65aS28PxooB6LdVnlExrmZ6JzbamY/ahR3NI5wzOvodDSuxicDG08b8Ixz7itNgjWbDJwCfAmYR2SwPhFARwrSvfUCKomMNBn9tbOO9i8iv6KFmY2h+SORDcBgM5vi1csys1He63OBAmA6sNgi4+jnEmngP7XIiLfn7Edcw8zsaO/1BcCLjd5/CTjJG5Uzem5kuLe+Xs65vwJX0Ux3mPRsOlKQ7uw1Ig3yW8BmIg14R/sVcL+ZbfDWtYHIXr/POVdnZrOBX3qNfjrwv2b2CZHzENOdc9vM7G4i50O+YWa/85a1nX2/+NceG4GrvZPe64EljWLaYWbfAP4YcxnvDUAN8Kh3HiSNyI++iPh0SapIKyzyoykZzrlar7vqb8Bwt++3jzsjps8Cf3LO6X4E6XA6UhBpXQGwzEsOBlzamQlBJNl0pCAiIj6daBYREZ+SgoiI+JQURETEp6QgIiI+JQUREfEpKYiIiO//Aygd+g6OZPx9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "random_forest = RandomForestClassifier(random_state=0, **best_parameters) \n",
    "train(random_forest, X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99       484\n",
      "           1       1.00      0.94      0.97        95\n",
      "\n",
      "   micro avg       0.99      0.99      0.99       579\n",
      "   macro avg       0.99      0.97      0.98       579\n",
      "weighted avg       0.99      0.99      0.99       579\n",
      "\n",
      "Accuracy: 0.9896373056994818\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAF3FJREFUeJzt3X20XXV95/H3xwAigqEYsJoEgjxUqaBoCjiupcyIFBkNLusDGdHiUNJaaXW0ts6qo5QWrTrasSOOpkJRVBAc60prLD4hGa1o4oAUUGhEHhJwAIEgAuHB7/yx9yXHS7LvyfXue07C+7XWXZy9z+/s/d0/bs7n7v0757dTVUiStCWPGXUBkqTxZlBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRTapiS5MsmRU7TZO8ndSebMUlm9S3JdkqPax6cm+dSoa9Kjh0GhGdG+kd3bvkH/vyRnJ9l1pvdTVb9ZVd+Yos0NVbVrVT000/tv36QfaI/zziT/kuS5M70faZwYFJpJL62qXYFnA4uBd0xukMa2/nv32fY45wEXAReMuJ4Zl2SHUdeg8bGt/4PVGKqq9cCXgGcAJPlGktOTfAu4B3hqkrlJzkxyc5L1Sf5q8FJRkpOT/CDJz5JcleTZ7frBSzCHJVmT5K72LOaD7fpFSWrizS7JU5KsSHJ7krVJTh7Yz6lJzk/yyXZfVyZZPORxPgh8GpifZM+Bbb4kyWUDZxyHDDy3MMnnk9ya5KdJPtyu3y/J19t1tyX5dJLdp9P/SY5r939Xkh8lOWZy3w0c+6cm9dlJSW4Avp7kS0lOmbTt7yd5efv4aUm+0vbr1UleNZ16Nf4MCs24JAuBY4FLB1a/FlgG7AZcD5wNPAjsDxwKHA38Xvv6VwKnAq8DngAsAX66mV19CPhQVT0B2A84fwslnQesA54CvAJ4d5L/MPD8krbN7sAK4MNDHudObY0/Be5o1x0KnAX8PvBE4GPAiiSPbYPwn9rjXwTMb/cLEOA9bY1PBxa2fbBVkhwGfBJ4W3s8zweu24pNvKDd/28D5wJLB7Z9ELAP8MUkjwe+AnwG2As4HvhI20bbGYNCM+kLSe4EvglcDLx74Lmzq+rK9q/wPWiC5M1V9fOqugX4G5o3G2gC431Vtboaa6vq+s3s7wFg/yTzquruqrpkcoM2tJ4H/FlV3VdVlwEfp3mDn/DNqlrZjmmcAzxziuN8VXuc9wInA69ojwuaMPxYVX2nqh6qqk8AG4EjgMNoguBt7XHfV1XfBGiP8StVtbGqbgU+SPOmvbVOAs5qt/WLqlpfVT/citef2tZ2L/APwLOS7NM+9xrg81W1EXgJcF1V/X1VPVhVlwL/G3jlNGrWmDMoNJNeVlW7V9U+VfWH7ZvNhBsHHu8D7Ajc3F6euZPmL++92ucXAj8aYn8nAQcCP0yyOslLNtPmKcDtVfWzgXXX0/w1P+EnA4/vAXZOskOS17SD1ncn+dJAm/OranfgScAVwHMmHdtbJ46rPbaFbR0LgesHQuVhSZ6U5Lz2MtxdwKdoxkC21rB9tyUP/39q++yLbArwpTSX2qA5zsMnHedrgF//FfatMeWAlWbL4DTFN9L8lT1vc2+a7fP7TbnBqn8DlraD4y8HPpfkiZOa3QTskWS3gbDYG1g/xPY/zaY3xs09f1uSZcCaJJ+pqpvb2k+vqtMnt28/HbV3kh02c9zvpumjg6vq9iQvY8hLYJN09d3PgV0Gljf3pj55OulzgXclWQXsTDN4P7Gfi6vqRdOoUdsYzyg069o31C8DH0jyhCSPaQdzJy61fBz4kyTPaT4klf0HLn88LMkJSfasql8Ad7arfzFpXzcC/wK8J8nO7cDySTR/sc/EsVwNXAj8abvq74A/SHJ4W/vjk/zHJLsB3wVuBv66Xb9zkue1r9sNuBvYkGQ+zRjDdJwJvD7JC9t+nZ/kae1zlwHHJ9mxHbB/xRDbW0lz9nAazae9Jvr3n4ADk7y23d6OSX4rydOnWbfGmEGhUXkdsBNwFc1A8OeAJwNU1QXA6TQDpT8DvkAzrjHZMcCVSe6mGdg+ftLlrglLaQaPb6K57v6uqvrqDB7L+4FlSfaqqjU04xYfbo9rLXAiQDsG8lKaAfwbaAbYX91u4y9oPla8geZyz+enU0hVfRd4Pc2YzwaasaKJkP1vNGcbd7T7+8wQ29vY1nLUYPv27OxomstSN9Fcvnsv8Njp1K3xFm9cJEnq4hmFJKmTQSFJ6mRQSJI6GRSSpE7b3Pco5s2bV4sWLRp1GZK0Tfne9753W1XtOXXLR9rmgmLRokWsWbNm1GVI0jYlyeamwRmKl54kSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUqfegiLJWUluSXLFFp5Pkr9Ncw/jy9PeE1mSNF76PKM4m2Ya6C15MXBA+7MM+F891iJJmqbevnBXVauSLOpochzwyWrmOb8kye5Jntze1GaL7rsPrrlmBguVpDGzxx4wbzo3wu3JKL+ZPZ9fvo/yunbdI4Kivd3kMoB5857KqlWzUp8kzbqNG5ugWLp01JVssk1M4VFVy4HlAAceuLgOPXTEBUlST66/Hm6/fdRV/LJRfuppPbBwYHkBQ9zwXpI0u0YZFCuA17WffjoC2DDV+IQkafb1dukpybnAkcC8JOuAdwE7AlTVR4GVwLE0N5+/h+aG8JKkMdPnp546h2LaTzu9sa/9S5Jmht/MliR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUqdegSHJMkquTrE3y9s08v3eSi5JcmuTyJMf2WY8kaev1FhRJ5gBnAC8GDgKWJjloUrN3AOdX1aHA8cBH+qpHkjQ9fZ5RHAasraprq+p+4DzguEltCnhC+3gucFOP9UiSpmGHHrc9H7hxYHkdcPikNqcCX07yR8DjgaM2t6Eky4BlAHvttfeMFypJ2rJRD2YvBc6uqgXAscA5SR5RU1Utr6rFVbV47tw9Z71ISXo06zMo1gMLB5YXtOsGnQScD1BV3wZ2Bub1WJMkaSv1GRSrgQOS7JtkJ5rB6hWT2twAvBAgydNpguLWHmuSJG2l3oKiqh4ETgEuBH5A8+mmK5OclmRJ2+ytwMlJvg+cC5xYVdVXTZKkrdfnYDZVtRJYOWndOwceXwU8r88aJEm/mlEPZkuSxpxBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI67TBswyTzgX0GX1NVq/ooSpI0PoYKiiTvBV4NXAU81K4uoDMokhwDfAiYA3y8qv56M21eBZzabu/7VfWfhi1ektS/Yc8oXgb8RlVtHHbDSeYAZwAvAtYBq5OsqKqrBtocAPxX4HlVdUeSvYYvXZI0G4Ydo7gW2HErt30YsLaqrq2q+4HzgOMmtTkZOKOq7gCoqlu2ch+SpJ4Ne0ZxD3BZkq8BD59VVNUfd7xmPnDjwPI64PBJbQ4ESPItmstTp1bVPw9ZkyRpFgwbFCvanz72fwBwJLAAWJXk4Kq6c7BRkmXAMoC99tq7hzIkSVsyVFBU1SeS7ER7BgBcXVUPTPGy9cDCgeUF7bpB64DvtNv6cZJraIJj9aT9LweWAxx44OIapmZJ0swYaowiyZHAv9EMTn8EuCbJ86d42WrggCT7tiFzPI88K/kCzdkESebRBNG1wxYvSerfsJeePgAcXVVXAyQ5EDgXeM6WXlBVDyY5BbiQZvzhrKq6MslpwJqqWtE+d3SSiY/dvq2qfjr9w5EkzbRhg2LHiZAAqKprkkz5KaiqWgmsnLTunQOPC3hL+yNJGkPDBsWaJB8HPtUuvwZY009JkqRxMmxQvAF4IzDxcdj/QzNWIUnazg37qaeNwAfbH0nSo0hnUCQ5v6peleRfaeZi+iVVdUhvlUmSxsJUZxRvav/7kr4LkSSNp87vUVTVze3D24Abq+p64LHAM4Gbeq5NkjQGhp0UcBWwc3tPii8DrwXO7qsoSdL4GDYoUlX3AC8HPlJVrwR+s7+yJEnjYuigSPJcmu9PfLFdN6efkiRJ42TYoHgzzQ2G/qGdhuOpwEX9lSVJGhfDfo/iYuDigeVr2fTlO0nSdmyq71H8j6p6c5J/ZPPfo1jSW2WSpLEw1RnFOe1//3vfhUiSxlNnUFTV99qHa4B7q+oXAEnm0HyfQpK0nRt2MPtrwC4Dy48Dvjrz5UiSxs2wQbFzVd09sdA+3qWjvSRpOzFsUPw8ybMnFpI8B7i3n5IkSeNk2PtRvBm4IMlNQIBfB17dW1WSpLEx7PcoVid5GvAb7aqrq+qB/sqSJI2LoS49JdkF+DPgTVV1BbAoiVOPS9KjwLBjFH8P3A88t11eD/xVLxVJksbKsEGxX1W9D3gAoJ1JNr1VJUkaG8MGxf1JHkc7jUeS/YCNvVUlSRobw37q6V3APwMLk3waeB5wYl9FSZLGx5RBkSTAD2luWnQEzSWnN1XVbT3XJkkaA1MGRVVVkpVVdTCbblokSXqUGHaM4v8m+a1eK5EkjaVhxygOB05Ich3wc5rLT1VVh/RVmCRpPAwbFL/daxWSpLE11R3udgb+ANgf+FfgzKp6cDYKkySNh6nGKD4BLKYJiRcDH+i9IknSWJnq0tNB7aedSHIm8N3+S5IkjZOpzigeniHWS06S9Og0VVA8M8ld7c/PgEMmHie5a6qNJzkmydVJ1iZ5e0e730lSSRZv7QFIkvrVeempquZMd8NJ5gBnAC8C1gGrk6yoqqsmtdsNeBPwnenuS5LUn2G/cDcdhwFrq+raqrofOA84bjPt/hJ4L3Bfj7VIkqapz6CYD9w4sLyuXfew9j7cC6uqc2qQJMuSrEmyZsOGW2e+UknSFvUZFJ2SPAb4IPDWqdpW1fKqWlxVi+fO3bP/4iRJD+szKNYDCweWF7TrJuwGPAP4Rjs1yBHACge0JWm89BkUq4EDkuybZCfgeGDFxJNVtaGq5lXVoqpaBFwCLKmqNT3WJEnaSr0FRfu9i1OAC4EfAOdX1ZVJTkuypK/9SpJm1rCTAk5LVa0EVk5a984ttD2yz1okSdMzssFsSdK2waCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSp16DIskxSa5OsjbJ2zfz/FuSXJXk8iRfS7JPn/VIkrZeb0GRZA5wBvBi4CBgaZKDJjW7FFhcVYcAnwPe11c9kqTp6fOM4jBgbVVdW1X3A+cBxw02qKqLquqedvESYEGP9UiSpqHPoJgP3DiwvK5dtyUnAV/a3BNJliVZk2TNhg23zmCJkqSpjMVgdpITgMXA+zf3fFUtr6rFVbV47tw9Z7c4SXqU26HHba8HFg4sL2jX/ZIkRwF/Drygqjb2WI8kaRr6PKNYDRyQZN8kOwHHAysGGyQ5FPgYsKSqbumxFknSNPUWFFX1IHAKcCHwA+D8qroyyWlJlrTN3g/sClyQ5LIkK7awOUnSiPR56YmqWgmsnLTunQOPj+pz/5KkX91YDGZLksZXr2cUkqStt3EjXHPNqKvYxKCQpDEydy785CewatVMb3m3x0/3lQaFJI2R3XeHww/vY8tz5kz3lY5RSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOvUaFEmOSXJ1krVJ3r6Z5x+b5LPt899JsqjPeiRJW6+3oEgyBzgDeDFwELA0yUGTmp0E3FFV+wN/A7y3r3okSdPT5xnFYcDaqrq2qu4HzgOOm9TmOOAT7ePPAS9Mkh5rkiRtpR163PZ84MaB5XXA4VtqU1UPJtkAPBG4bbBRkmXAsnbpgcWLf+26Xire5mycC4/dMOoqxoN9sYl9sYl9scldC6b7yj6DYsZU1XJgOUCSNVV3LB5xSWOh6Yt77Avsi0H2xSb2xSZJ1kz3tX1eeloPLBxYXtCu22ybJDsAc4Gf9liTJGkr9RkUq4EDkuybZCfgeGDFpDYrgN9tH78C+HpVVY81SZK2Um+Xntoxh1OAC4E5wFlVdWWS04A1VbUCOBM4J8la4HaaMJnK8r5q3gbZF5vYF5vYF5vYF5tMuy/iH/CSpC5+M1uS1MmgkCR1GtugcPqPTYboi7ckuSrJ5Um+lmSfUdQ5G6bqi4F2v5Okkmy3H40cpi+SvKr93bgyyWdmu8bZMsS/kb2TXJTk0vbfybGjqLNvSc5KckuSK7bwfJL8bdtPlyd59lAbrqqx+6EZ/P4R8FRgJ+D7wEGT2vwh8NH28fHAZ0dd9wj74t8Du7SP3/Bo7ou23W7AKuASYPGo6x7h78UBwKXAr7XLe4267hH2xXLgDe3jg4DrRl13T33xfODZwBVbeP5Y4EtAgCOA7wyz3XE9o3D6j02m7Iuquqiq7mkXL6H5zsr2aJjfC4C/pJk37L7ZLG6WDdMXJwNnVNUdAFV1yyzXOFuG6YsCntA+ngvcNIv1zZqqWkXzCdItOQ74ZDUuAXZP8uSptjuuQbG56T/mb6lNVT0ITEz/sb0Zpi8GnUTzF8P2aMq+aE+lF1bVF2ezsBEY5vfiQODAJN9KckmSY2atutk1TF+cCpyQZB2wEvij2Slt7Gzt+wmwjUzhoeEkOQFYDLxg1LWMQpLHAB8EThxxKeNiB5rLT0fSnGWuSnJwVd050qpGYylwdlV9IMlzab6/9Yyq+sWoC9sWjOsZhdN/bDJMX5DkKODPgSVVtXGWapttU/XFbsAzgG8kuY7mGuyK7XRAe5jfi3XAiqp6oKp+DFxDExzbm2H64iTgfICq+jawMzBvVqobL0O9n0w2rkHh9B+bTNkXSQ4FPkYTEtvrdWiYoi+qakNVzauqRVW1iGa8ZklVTXsytDE2zL+RL9CcTZBkHs2lqGtns8hZMkxf3AC8ECDJ02mC4tZZrXI8rABe13766QhgQ1XdPNWLxvLSU/U3/cc2Z8i+eD+wK3BBO55/Q1UtGVnRPRmyLx4VhuyLC4Gjk1wFPAS8raq2u7PuIfvircDfJfkvNAPbJ26Pf1gmOZfmj4N57XjMu4AdAarqozTjM8cCa4F7gNcPtd3tsK8kSTNoXC89SZLGhEEhSepkUEiSOhkUkqROBoUkqZNBIU2S5KEklyW5Isk/Jtl9hrd/YpIPt49PTfInM7l9aaYZFNIj3VtVz6qqZ9B8R+eNoy5IGiWDQur2bQYmTUvytiSr27n8/2Jg/evadd9Pck677qXtvVIuTfLVJE8aQf3Sr2wsv5ktjYMkc2imfTizXT6aZq6kw2jm81+R5Pk0c4y9A/h3VXVbkj3aTXwTOKKqKsnvAX9K8w1haZtiUEiP9Lgkl9GcSfwA+Eq7/uj259J2eVea4HgmcEFV3QZQVRP3A1gAfLad738n4MezU740s7z0JD3SvVX1LGAfmjOHiTGKAO9pxy+eVVX7V9WZHdv5n8CHq+pg4PdpJqKTtjkGhbQF7V0D/xh4azuV/YXAf06yK0CS+Un2Ar4OvDLJE9v1E5ee5rJpCuffRdpGeelJ6lBVlya5HFhaVee0U1R/u52l927ghHam0tOBi5M8RHNp6kSau6pdkOQOmjDZdxTHIP2qnD1WktTJS0+SpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnq9P8BZrz6LMLWEJMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluate(random_forest, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameter tuning with GridsearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 14 candidates, totalling 42 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best options\n",
      "=======================================\n",
      "C: 100.0\n",
      "penalty: 'l2'\n",
      "solver: 'liblinear'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Done  42 out of  42 | elapsed:    2.3s finished\n"
     ]
    }
   ],
   "source": [
    "param_grid={\"C\":np.logspace(-3,3,7), \"penalty\":[\"l1\",\"l2\"], \"solver\":[\"liblinear\"]} # l1 lasso l2 ridge\n",
    "best_parameters = hyper_parameter_tuning(param_grid , LogisticRegression(random_state=0), 3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training with best parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VOXd9/HPb7KHLBARVBBCLcoWliTFFYUH9cY+rdZdClbcolSt1kef4nJbb1vu0s1qrVXxsd7VRqlUqVh3KdRSBVmKICCCCsqiyBYIIYEk1/PHnDlMkkkygVkS+L5fr/Oac665zpnfXJmc3znXOXONOecQEREBCCQ7ABERaT+UFERExKekICIiPiUFERHxKSmIiIhPSUFERHxKCiIi4lNSEBERX9ySgpn9wcw2m9kHzTxvZvZbM1tjZkvNrDhesYiISHRS47jt/wF+BzzVzPPnAH296UTgEe+xRV27dnWFhYVti2TbNtiwAfbuhfR06NEDCgpaXW337t106tSpba8VhzjaqtW4ExRHqxrFUX3EEWQec0xSY2hrW8TsM5Lgv0mzcbfTz0Yojpj/Tx5gHG0Rk5hjEMeiRYu2OOeObLWicy5uE1AIfNDMc48BY8OWVwFHt7bNkpIS1yZ/+pNz2dnOwf4pOztY3orZs2e37bXiFEdbtRh3AuNoUYQ4ajMyEhtHDNoiJp+RJPxNIsbdjj8boThi+j95EHG0xUHHHKM4gIUuiv22uTiOfWRmhcDfnHODIjz3N2CKc26utzwL+JFzbmFL2ywtLXULF7ZYpaHCQli3rml5RgacdFKLq+7YsYPOnTtH/1otmTcPamoOKI62ajHuBMbRovYQRwxiiMlnJAltETHu9vA3aSWOHf36xe5/8iDiaEt7HPRnpLk4eveGtWuj3oyZLXLOlbZWL57dRzFjZmVAGUD37t2ZM2dO1Oue8dlnWIRyV1NDxfbtLa5bV1/PjlbqRCu/puaA42irluJOZBwtaQ9xxCKGWHxGktEWkeJuD3+T1uKoq6uL2f/kwcTRlvY42M9Is3F89hn/aMO+MGrRnE4c6ER76D7q3bvhaVdo6t271VVjeqp6EHG0VYtxJzCOFrWHOGIQQ0w+I0loi4hxt4e/SStxJLT7KEbtcdAxxygOouw+SuYtqTOB73l3IZ0EVDjnNsX8VSZPhuzshmXZ2cHyRFIcrcZRl5GR2DjacVskK47ykjQKb4HAj6HwFigvSVMch1kcces+MrNngZFAVzNbD/wYSANwzj0KvAJ8E1gDVAFXxiWQceOCj3fdBZ99Br16BRszVJ4oiqPVOFaNH8+ARMbRjtqifOe/uOuTqXzWqY5eu1OY/LUrGJfgOMoHQ9m5RpV3mXFd5+AygyGRkbQUR4+t7SOOxu3hnMPh/Mf6+nrqqae2rpaa+hq2VW2j1tVSWx+c6l09e2v3UltfS52rY1/dvgaPoXp19XW8deS/+d23HaGrCvH+u8T1QnM8tPlC80GYM2cOI0eOTMhrxZLiTpxYxFy+rJyyl8qo2lfll2WnZTP121MZV9S2f/va+lqqa6uprq2mprbGn6+uraambv/ywiULOe6E46iu857fV81P3v4JFTUVTbaZk57DZQMvo97VU+fq9j/Why3X1zV8vpnlFut4j59u/5Q6V9ckjhRLoWt6V9Iy0treyAdg065NEeMIWID8jHw/3lACCL2/0BRvvfN7s/aWtVHXP6QuNMuhqXxZOXfNuovPKj6jV34vxh89npGMTGoMk0dPbvOOOFp19XXsqN7RYKqoruDmV29ukBAAqvZVUfZSGTNWzmiwMw/t6PfW7fV39DW1Nf5jpJ1Ys1ZEV61ybyUvfPgCKZaCmUX9GLBAg8kwUgIppFgKaSlpBPCeCwQarLtm25rI7efqKMovIrtzdsTnY23mzpkRy+tdPed8/RwCFiA1kErAAqQEUjDMnw9Y8D0FLMD2Tdvp1rPb/ufYXye8Xmog1W8Dv90CAa7/2/UR4/is4rO4vG8lBUmKPy39E2UvlbGndg8A6yrW8ctdv6Tb/G5cMvAS0gPppAZSSUtJ83cYgP8YziLemxG5bnj9Z5Y9Q9nf9h+hr6tYR9lLZdTX1zO2aCzOOepdvd8dsGvvruCOvKaCiuoKdlTvYMmXS1jwrwV+2c6aneys2UlFTXB+195d7KrZxc6anezet7tNbVS1r4pFGxeRnppOekpwykjJoCCroMFyRmrG/uXUDDJSMvz5zJTMBvUyUzPJTM1k8+rNHF90PJmpmWSlZpGVlsWZT5/Jxl0bm8TRM68nS69f2qQ8YC1fkjSzZv824XUaG/DwAD7f+XmT8mPzjuWOfncw/NThLW4zVlqK4/f/+/cADc4KIj3Wu3pWLFxBv5J+TZ4L9dLU45XVe2cXoSZx4HAck3tMxL9Lr/xeMXiXTSkpHIbicXRcXVvN5srNbN7tTVWb+Wr3V2yp2sLm3ZvZUrWFLVVb2LpnK1v3bGXbnm1NtrG3fi83v3YzN792c5Pn0gJppAZSG0x+WUpqk+caPB+2nBIIHqWmBlJ5adVLEY/Qr33pWh567yF/h75r7y4q91Y23yXw4f4YczNyyUvPIzcjl9z0XLp27kpOeg55GXnkpOcEp7Tgcm56LjkZOXz/5e+zuWpzk832yO3B3KvmNjnibnD07e142/IIMGfzHEZ+bWSD1/vFWb+I2I015cwpdMnq0sJfP7Z+dubPIsbxszN/RmBrgJz0nKTHkZ+ZH/V2VgdWc2z+sVHXDyWL0DWKKaOncP3L1zeJY/LoDnahWdqnxv3XoaNjwE8M9a6eHdU7gjv0yuAOfvPu4E7+q6qvgjv7PVvYsnv/Tr7xzjUkYAE6Z3amS2YXCrIK6NOlD8VHF/Pn5X9uNsa7R9ztX2irra8NXqCrazTfuCxUt66WffX7qK2vZU/tHmrra9lXF1zeV7+Puvo6//nmjtxr6mpIDaTSK7+Xv/POzcj1d+q56bn+zn3HxzvoP6Q/uRm5ZKRkNDjyTQ2kkhJIaZKswrsOAhagal9VxH/6n5/1c3rk9WjbH/gghP7+iepOO5A42vIdpXjGEU/+WTEGBpcPuZxAIJCwOHShuQWxvvCZqP7rUNzOOar2VbG9ejvbqraxvXo7F0+/mK+qvmqyTkZKBn269GFL1Ra279nebN90VmoWXbK60DmzMwVZBf7OvktmF47IOoLOWZ0pyCygS3YXCjILyMvI87uAQv3JqYFUSh8vZf3O9U22f2zesSybuKzJRbvGp+ShKfwUvL6+PnjqHfpIh+a9R4cL/sN5ZaP+ZxQbK5uelh+Tcwyzr5jdoKy5Hfyidxdx0mknRTyKb4tEXtuAjnlRHzpm3O0lZl1obmeiOUJvfFubc47a+loqqivYtie4U99evZ0d1TvYvsd7rN7uz++o3sGOmh1sqdjCnvf2sLNmJ/vq90UVX01dDT1yezDoyEF0yQru4AuyCvz50GNOeo5/ca25o9/QBbfwLo7Gppw5pcmpeUYgo82n5o2F2q659oT9p+U//V8/5YZXbvCva0Aw6U0ePZnCLoVR7eADFiA77eAvfI4rGpfwI3KRSJQUEuTOWXdG7L+e+LeJzP50dvDCZPX+C5ShqXJvpb+TiyQ1kEpuei75mfnkZeSRl55Hfqd8jup+lF+enxF8Lj8jn9vfup0tVVuabKdnXk+ev+T5Nu/cD1SkU/PxR48/6B1jg4ubrYR75bArSU9NT3p3iUh7oqQQJ/WunnU71vHu5+/y7vp3m719bNfeXTyz7Bl/552fmU/PvJ7BHbw35abn+hcq8zKCFzFDO/rstGz/NrbQUfvKhSspObnEP6L3b30LpJCaksp1f7su4sXEgzlCPxCNj44T2V/cXAwihzslhRjZvmc7765/l3nr5/Hehvd4/8v3+aLyCwD/zpdIXTlH5xzNnAlzCD8ZcLgG/e+h9dNS0prcAx3pKH51YHWzd4uMHzweM9PRsYhEpKRwAGpqa1j8xWLe/fxd5q+fz+JNi/l4+8d+N09hfiEn9TyJId2HMPDIgZxwxAm8+cmb3D37bqprq/3tZKVm8ZNRP6FHbo8md6S09UJlW+joWESao6TQinpXz6otq3h3fTABLNi4gOWbl7O3fi8AXbO7MqT7EM494VwGdR/EwK4Dyc/Mx8xID6STnZ5NVmoWNwy/gW6duvGfs/9TR+gi0m4pKTSyYecG5m+Yz7z185i1Yhar313Nrr27gGDf+6Bug7hi6BUUdSticLfBHJV7FKmBVDJTM8lOyyY9JZ20lDTSAmlNLsxePuRyLh9yeTLelohIVA6LpNDcPeA7qnewcONC3tvwHvPWz2PhxoVsqgyO3p0aSKVPVh++2febDO42mMHdB3NcwXFkp2WTlZZFVmqWv/NPCaQk+R2KiMTGIZ8UIn0/4IoZV3Db67fxxe4v/HqF+YUMPWooV3S/gqFHDaWoWxEbPthA6cmlLR79i4gcSg75pHDXrLuafD+gztWxvXo7Nw2/iaFHDaXkqBKOzju6ydH/5sDmhI75IiKSbId8Umju+wF76/byq7N/paN/EZEwyfw5zoRobnjZXvm9SE9JV0IQEQlzyCeFyaMnNxmbJp7DzoqIdGSHfFIYVzSOqd+eSu/83hhG7/zeB/QzhyIih4ND/poC6Bu8IiLROuTPFEREJHpKCiIi4lNSEBERn5KCiIj4lBRERMSnpCAiIj4lBRER8SkpiIiIT0lBRER8SgoiIuJTUhAREZ+SgoiI+JQURETEp6QgIiI+JQUREfEpKYiIiE9JQUREfEoKIiLiU1IQERFfXJOCmY0xs1VmtsbMJkV4vpeZzTazf5vZUjP7ZjzjERGRlsUtKZhZCvAwcA4wABhrZgMaVbsbeM45Nwy4DPh9vOIREZHWxfNMYTiwxjn3iXNuLzANOK9RHQfkefP5wMY4xiMiIq1IjeO2ewCfhy2vB05sVOde4A0zuwnoBJwZx3hERKQV5pyLz4bNLgLGOOeu8ZYvB050zt0YVudWL4Zfm9nJwBPAIOdcfaNtlQFlAN27dy+ZNm1aXGJurLKykpycnIS8Viwp7sTpiDGD4k6k9hLzqFGjFjnnSlut6JyLywScDLwetnwHcEejOsuBY8OWPwG6tbTdkpISlyizZ89O2GvFkuJOnI4Ys3OKO5HaS8zAQhfFvjue1xQWAH3NrI+ZpRO8kDyzUZ3PgNEAZtYfyAS+imNMIiLSgrglBedcLXAj8DqwkuBdRsvN7D4zO9er9n+Aa83sfeBZYIKX0UREJAnieaEZ59wrwCuNyu4Jm18BnBrPGEREJHr6RrOIiPiUFERExKekICIiPiUFERHxKSmIiIhPSUFERHxKCiIi4lNSEBERn5KCiIj4lBRERMSnpCAiIj4lBRER8SkpiIiIT0lBRER8SgoiIuJTUhAREZ+SgoiI+JQURETEp6QgIiI+JQUREfEpKYiIiE9JQUREfEoKIiLiU1IQERGfkoKIiPiUFERExKekICIiPiUFERHxKSmIiIhPSUFERHxKCiIi4lNSEBERn5KCiIj4lBRERMSnpCAiIj4lBRER8SkpiIiIT0lBRER8SgoiIuKLa1IwszFmtsrM1pjZpGbqXGJmK8xsuZk9E894RESkZanx2rCZpQAPA2cB64EFZjbTObcirE5f4A7gVOfcdjPrFq94RESkdVGfKZjZaWZ2pTd/pJn1aWWV4cAa59wnzrm9wDTgvEZ1rgUeds5tB3DObY4+dBERiTVzzrVeyezHQClwgnPueDM7BpjunDu1hXUuAsY4567xli8HTnTO3RhW56/AR8CpQApwr3PutQjbKgPKALp3714ybdq0NrzFA1dZWUlOTk5CXiuWFHfidMSYQXEnUnuJedSoUYucc6Wt1Yu2++h8YBiwGMA5t9HMcg8ivvDX7wuMBHoCb5tZkXNuR3gl59xUYCpAaWmpGzlyZAxeunVz5swhUa8VS4o7cTpizKC4E6mjxRxt99FeFzylcABm1imKdTYAx4Yt9/TKwq0HZjrn9jnnPiV41tA3yphERCTGok0Kz5nZY0BnM7sWeAt4vJV1FgB9zayPmaUDlwEzG9X5K8GzBMysK3A88EmUMYmISIxF1X3knPuVmZ0F7AROAO5xzr3Zyjq1ZnYj8DrB6wV/cM4tN7P7gIXOuZnec2eb2QqgDrjdObf1IN6PiIgchFaTgndr6VvOuVFAi4mgMefcK8ArjcruCZt3wK3eJCIiSdZq95Fzrg6oN7P8BMQjIiJJFO3dR5XAMjN7E9gdKnTO/SAuUYmISFJEmxRe8CYRETmERXuh+Y/eHUTHe0WrnHP74heWiIgkQ1RJwcxGAn8E1gIGHGtmVzjn3o5faCIikmjRdh/9GjjbObcKwMyOB54FSuIVmIiIJF60X15LCyUEAOfcR0BafEISEZFkifZMYaGZ/T/gT97yOGBhfEISEZFkiTYpTARuAEK3oP4T+H1cIhIRkaSJNimkAg865+4H/1vOGXGLSkREkiLaawqzgKyw5SyCg+KJiMghJNqkkOmcqwwtePPZ8QlJRESSJdqksNvMikMLZlYK7IlPSCIikizRXlO4BZhuZhu95aOBS+MTkoiIJEuLZwpm9g0zO8o5twDoB/wZ2Ae8BnyagPhERCSBWus+egzY682fDNwJPAxsx/vNZBEROXS01n2U4pzb5s1fCkx1zj0PPG9mS+IbmoiIJFprZwopZhZKHKOBv4c9F+31CBER6SBa27E/C/zDzLYQvNvonwBm9nWgIs6xiYhIgrWYFJxzk81sFsG7jd7wflMZgmcYN8U7OBERSaxWu4Ccc/MilH0Un3BERCSZov3ymoiIHAaUFERExKekICIiPiUFERHxKSmIiIhPSUFERHxKCiIi4lNSEBERn5KCiIj4lBRERMSnpCAiIj4lBRER8SkpiIiIT0lBRER8SgoiIuJTUhAREZ+SgoiI+OKaFMxsjJmtMrM1ZjaphXoXmpkzs9J4xiMiIi2LW1IwsxTgYeAcYAAw1swGRKiXC9wMzI9XLCIiEp14nikMB9Y45z5xzu0FpgHnRaj3E+DnQHUcYxERkSjEMyn0AD4PW17vlfnMrBg41jn3chzjEBGRKKUm64XNLADcD0yIom4ZUAbQvXt35syZE9fYQiorKxP2WrGkuBOnI8YMijuROlzMzrm4TMDJwOthy3cAd4Qt5wNbgLXeVA1sBEpb2m5JSYlLlNmzZyfstWJJcSdOR4zZOcWdSO0lZmChi2LfHc/uowVAXzPrY2bpwGXAzLBkVOGc6+qcK3TOFQLzgHOdcwvjGJOIiLQgbknBOVcL3Ai8DqwEnnPOLTez+8zs3Hi9roiIHLi4XlNwzr0CvNKo7J5m6o6MZywiItI6faNZRER8SgoiIuJTUhAREZ+SgoiI+JQURETEp6QgIiI+JQUREfEpKYiIiE9JQUREfEoKIiLiU1IQERGfkoKIiPiUFERExKekICIiPiUFERHxKSmIiIhPSUFERHxKCiIi4lNSEBERn5KCiIj4lBRERMSnpCAiIj4lBRER8SkpiIiIT0lBRER8SgoiIuJTUhAREZ+SgoiI+JQURETEl5rsAGJh3759rF+/nurq6phuNz8/n5UrV8Z0m4nQ0ePOzMykZ8+epKWlJTskkcPOIZEU1q9fT25uLoWFhZhZzLa7a9cucnNzY7a9ROnIcefk5LB161bWr19Pnz59kh2SyGHnkOg+qq6u5ogjjohpQpDkMDOOOOKImJ/1iUh0DomkACghHEL0txRJnkMmKSTT1q1bGTp0KEOHDuWoo46iR48e/vLevXuj2saVV17JqlWrWqzz8MMPU15eHouQRUQiOiSuKbRZeTncdRd89hn06gWTJ8O4cQe8uSOOOIIlS5YAcO+995KTk8Ntt93WoI5zDuccgUDkPPzkk0+2+jo33HDDAccYT629NxHpOA6//+Lycigrg3XrwLngY1lZsDzG1qxZw4ABAxg3bhwDBw5k06ZNlJWVUVpaysCBA7nvvvv8uqeddhpLliyhtraWzp07M2nSJIYMGcLJJ5/M5s2bAbj77rt54IEH/PqTJk1i+PDhnHDCCbzzzjsA7N69m/HjxzNgwAAuuugiSktL/YQV7vbbb2fAgAEMHjyYH/3oRwB88cUXnHfeeQwePJghQ4Ywf/58AH7xi18waNAgBg0axEMPPdTse3v11Vc5+eSTKS4u5tJLL2X37t0xb1MRia9D70zhllsgwk7QN28e1NQ0LKuqgquvhscfb1CcVVcHKSkwdCh4O+O2+vDDD3nqqacoLS0FYMqUKRQUFFBbW8uoUaO46KKLGDBgQIN1KioqOOOMM5gyZQq33norf/jDH5g0aVKTbTvneO+995g5cyb33Xcfr732Gg899BDdunXjxRdf5P3336e4uLjJel9++SWvvPIKy5cvx8zYsWMHEDwTOeuss7jxxhupra2lqqqK+fPnU15ezoIFC6itrWX48OGMHDmSrKysBu9t8+bNTJkyhVmzZpGdnc3kyZN58MEHufPOOw+o3UQkOQ6/M4XGCaG18oN03HHH+QkB4Nlnn6W4uJji4mJWrlzJihUrmqyTlZXFOeecA0BJSQlr166NuO0LLrigSZ25c+dy0UUXATBkyBAGDhzYZL2CggICgQDXXnstM2bMoFOnTgDMmTOH6667DoDU1FTy8vKYO3cuF154IVlZWeTm5vKd73yHf/7zn03e2zvvvMOKFSs45ZRTGDp0KOXl5c3GLSLt16F3ptDaEX1hYbDLqLHevWHOnAZFe2Jwv39ohwuwevVqHnzwQd577z06d+7M+PHjI956mZ6e7s+npKRQW1sbcdsZGRmt1okkLS2NhQsX8uabbzJ9+nQeeeQR3njjDaBtd/6EvzfnHGPGjOHpp5+Oen0RaX/ieqZgZmPMbJWZrTGzJv0fZnarma0ws6VmNsvMesczHiB4UTk7u2FZdnawPM527txJbm4ueXl5bNq0iddffz3mr3HqqafywgsvALBs2bKIZyK7du1i586dfOtb3+I3v/kN//73vwEYNWoUjz76KAB1dXXs3LmTESNGMGPGDPbs2UNlZSUvvvgiI0aMaLLNU045hX/84x988sknQPDaxurVq2P+/kQkvuJ2pmBmKcDDwFnAemCBmc10zoXvpf4NlDrnqsxsIvAL4NJ4xQTsv8sohncfRau4uJgBAwbQr18/evfuzamnnhrz17jpppv47ne/y4ABA/wpPz+/QZ2KigouuOACampqqK+v5/777wfgd7/7Hddeey2PPfYYqampPPbYYwwfPpyxY8fyjW98A4CJEydSVFTEmjVrGmyze/fuPPHEE1x66aX+bbj//d//Td++fWP+HkUkjkK3E8Z6Ak4GXg9bvgO4o4X6w4B/tbbdkpIS19iKFSualMXCzp0747LdeNq3b5/bvHmzc865jz76yBUWFrp9+/YlOarohLd3vP6msTZ79uxkh3BAFHfitJeYgYUuin13PK8p9AA+D1teD5zYQv2rgVfjGM9hobKykrPPPpv6+nqcc/5Rv4hINNrF3sLMxgOlwBnNPF8GlEGwm2JOowvC+fn57Nq1K+Zx1dXVxWW78ZSSksLs2bNJSUnxyzrKewhv7+rq6iZ/5/aosrKyQ8TZmOJOnI4WczyTwgbg2LDlnl5ZA2Z2JnAXcIZzLuJ9oc65qcBUgNLSUjdy5MgGz69cuTIuo4J25NFGO3rcmZmZDBs2LMkRtW7OnDk0/jx2BIo7cTpazPG8+2gB0NfM+phZOnAZMDO8gpkNAx4DznXObY5jLCIiEoW4JQXnXC1wI/A6sBJ4zjm33MzuM7NzvWq/BHKA6Wa2xMxmNrM5ERFJgLheU3DOvQK80qjsnrD5M+P5+iIi0jaH3zAXcTBq1KgmX0R74IEHmDhxYovr5eTkALBx40Z/aIrGRo4cycKFC1vczgMPPEBVVZW/fOGFF/rjGYmItMVhmRTKl5VT+EAhgf8KUPhAIeXLDm6E1LFjxzJt2rQGZdOmTWPs2LFRrX/MMcfwl7/85YBfv3FSeP755+ncufMBby9e6urqkh2CiLTisEsK5cvKKXupjHUV63A41lWso+ylsoNKDBdddBEvv/yy/03etWvXsnHjRkaMGEFlZSWjR4+muLiYoqIiXnzxxSbrr127lkGDBgGwZ88eLrvsMvr378/555/Pnj17/HoTJ070h93+8Y9/DMBvf/tbNm7cyKhRoxg1ahQAgwYNYsuWLQDcf//9/rDXoWG3165dS//+/bn22msZOHAgZ599doPXCZk+fTqDBg1iyJAhnH766UBwx37bbbcxaNAgBg8e7A+lPWvWLIYNG0ZRURFXXXUVNd4Ag4WFhfzoRz+iuLiY6dOn8/HHHzNmzBhKSkoYMWIEH3744QG3u4jEXrv4nkIs3fLaLSz5ovmhs+etn0dNXcM7X6v2VXH1i1fz+KKGQ2fX1dWRkpLC0KOG8sCY5gfaKygoYPjw4bz66qucd955TJs2jUsuuQQzIzMzkxkzZpCXl8eWLVs46aSTOPfcc5sdeO6RRx4hOzublStXsnTp0gZDX0+ePJmCggLq6uoYPXo0S5cu5Qc/+AH3338/s2fPpmvXrg22tWjRIp588knmz5+Pc44TTzyRM844gy5durB69WqeffZZHn/8cS655BKef/55xo8f32D9++67j9dff50ePXr43VFTp05l7dq1LFmyhNTUVLZt20Z1dTUTJkxg1qxZHH/88Xzve9/jkUce4ZZbbgGCP0K0ePFiAEaPHs2jjz5K3759mT9/Pt///vf5+9//3mzbikhiHXZnCo0TQmvl0QrvQgrvOnLOceeddzJ48GDOPPNMNmzYwJdfftnsdt5++21/5zx48GAGDx7sP/fcc89RXFzMsGHDWL58ecTB7sLNnTuX888/n06dOpGTk8MFF1zgD3vdp08fhg4dCjQ/PPepp57KhAkTePzxx/2un7feeovrrrvO/5Z0QUEBq1atok+fPhx//PEAXHHFFbz99tv+di69NDicVWVlJe+88w4XX3wxQ4cO5brrrmPTpk0tvgcRSaxD7kyhpSN6gMIHCllX0XTo7N75vZkzYU6DsrZ8Cey8887jhz/8IYs6Av/yAAALg0lEQVQXL6aqqoqSkhIAysvL+eqrr1i0aBFpaWkUFhZGHC67NZ9++im/+tWvWLBgAV26dGHChAkHtJ2Q0LDbEPwWdKTuo0cffZT58+fz8ssvU1JSwqJFiw7otUJDbNfX19O5c+eIvwQnIu3DYXemMHn0ZLLTGg6dnZ2WzeTRBzd0dk5ODqNGjeKqq65qcIG5oqKCbt26kZaWxuzZs1kX6bccwpx++uk888wzAHzwwQcsXboUCA673alTJ/Lz8/nyyy959dX9w0Tl5uZGHMpixIgR/PWvf6Wqqordu3czY8aMiMNeN+fjjz/mxBNP5L777uPII4/k888/56yzzuKxxx7zf79h27ZtnHDCCaxdu9YfOfXpp5/mjDOajliSl5dHnz59mD59OhA8i3r//fejjkdE4u+wSwrjisYx9dtT6Z3fG8Pond+bqd+eyriigx86e+zYsbz//vsNksK4ceNYuHAhRUVFPPXUU/Tr16/FbUycOJHKykr69+/PPffc459xDBkyhGHDhtGvXz+++93vNhh2u6ysjDFjxvgXmkOKi4uZMGECw4cP58QTT+Saa65p09ARt99+O0VFRQwaNIhTTjmFIUOGcM0119CrVy//d5yfeeYZMjMzefLJJ7n44ospKioiEAhw/fXXR9xmeXk5TzzxhP+rcJEuvItI8lhwRNWOo7S01DW+b3/lypX0798/5q91KIwh1JGExx2vv2msdbRxbUIUd+K0l5jNbJFzrrS1eofdmYKIiDRPSUFERHxKCiIi4jtkkkJHuzYizdPfUiR5DomkkJmZydatW7UzOQQ459i6dSuZmZnJDkXksHRIfHmtZ8+erF+/nq+++iqm262uru6QO6eOHndmZiY9e/ZMdjgih6VDIimkpaXRp0+fmG93zpw5HeInIRtT3CJyoA6J7iMREYkNJQUREfEpKYiIiK/DDXNhZl8BLY8qFztdgS0Jeq1YUtyJ0xFjBsWdSO0l5t7OuSNbq9ThkkIimdnCaMYKaW8Ud+J0xJhBcSdSR4tZ3UciIuJTUhAREZ+SQsumJjuAA6S4E6cjxgyKO5E6VMy6piAiIj6dKYiIiO+wTQpmdqyZzTazFWa23Mxu9srvNbMNZrbEm74Zts4dZrbGzFaZ2X8kMfa1ZrbMi2+hV1ZgZm+a2WrvsYtXbmb2Wy/upWZWnKSYTwhr0yVmttPMbmmP7W1mfzCzzWb2QVhZm9vXzK7w6q82syuSEPMvzexDL64ZZtbZKy80sz1hbf5o2Dol3mdrjfe+LAlxt/kzYWZjvLI1ZjYpnjG3EPefw2Jea2ZLvPJ2095Rcc4dlhNwNFDszecCHwEDgHuB2yLUHwC8D2QAfYCPgZQkxb4W6Nqo7BfAJG9+EvBzb/6bwKuAAScB89tB26cAXwC922N7A6cDxcAHB9q+QAHwiffYxZvvkuCYzwZSvfmfh8VcGF6v0Xbe896Hee/rnCS0dZs+E970MfA1IN2rMyDRcTd6/tfAPe2tvaOZDtszBefcJufcYm9+F7AS6NHCKucB05xzNc65T4E1wPD4Rxq184A/evN/BL4TVv6UC5oHdDazo5MRYJjRwMfOuZa+hJi09nbOvQ1sixBPW9r3P4A3nXPbnHPbgTeBMYmM2Tn3hnOu1lucB7Q49KwXd55zbp4L7rGeYv/7jItm2ro5zX0mhgNrnHOfOOf2AtO8unHTUtze0f4lwLMtbSMZ7R2NwzYphDOzQmAYMN8rutE75f5DqJuAYML4PGy19bScROLJAW+Y2SIzK/PKujvnNnnzXwDdvfn2FHfIZTT8h2nv7Q1tb9/2Fv9VBI9EQ/qY2b/N7B9mNsIr60EwzpBkxtyWz0R7a+sRwJfOudVhZe29vX2HfVIwsxzgeeAW59xO4BHgOGAosIngaWB7c5pzrhg4B7jBzE4Pf9I76miXt5WZWTpwLjDdK+oI7d1Ae27fSMzsLqAWKPeKNgG9nHPDgFuBZ8wsL1nxRdDhPhONjKXhQU97b+8GDuukYGZpBBNCuXPuBQDn3JfOuTrnXD3wOPu7LDYAx4at3tMrSzjn3AbvcTMwg2CMX4a6hbzHzV71dhO35xxgsXPuS+gY7e1pa/u2i/jNbALwLWCcl8zwul+2evOLCPbHH+/FF97FlJSYD+Az0S7aGsDMUoELgD+Hytp7ezd22CYFr9/vCWClc+7+sPLw/vbzgdDdBTOBy8wsw8z6AH0JXiRKKDPrZGa5oXmCFxM/8OIL3eFyBfCiNz8T+J53l8xJQEVYN0gyNDiKau/tHaat7fs6cLaZdfG6P872yhLGzMYA/xc41zlXFVZ+pJmlePNfI9i2n3hx7zSzk7z/j++x/30mMu62fiYWAH3NrI93JnqZVzcZzgQ+dM753ULtvb2bSPaV7mRNwGkEuwCWAku86ZvA08Ayr3wmcHTYOncRzPKrSNJdAgTvsHjfm5YDd3nlRwCzgNXAW0CBV27Aw17cy4DSJLZ5J2ArkB9W1u7am2DS2gTsI9jPe/WBtC/Bfvw13nRlEmJeQ7CvPfT5ftSre6H32VkCLAa+HbadUoI74Y+B3+F9wTXBcbf5M+H9737kPXdXMj4jXvn/ANc3qttu2juaSd9oFhER32HbfSQiIk0pKYiIiE9JQUREfEoKIiLiU1IQERGfkoK0O2Z2RNiIkl80GjEzPcptPGlmJ7RS5wYzGxebqNsHM5trZkOTHYd0XLolVdo1M7sXqHTO/apRuRH8/NYnJbB2yszmAjc655YkOxbpmHSmIB2GmX3dgr9/UU7wy0BHm9lUM1towd/EuCes7lwzG2pmqWa2w8ymmNn7ZvaumXXz6vzUzG4Jqz/FzN6z4Lj8p3jlnczsee91/+K9VpMjcTP7hjfY2SIze9XMuptZmrd8mlfnl2b2X978f5nZAjP7wMwe9ZJcKI77vddZYWalFvwthNVeggy1w3Izm2ZmK83sOTPLihDTOd77XWzBsf47hcWxwoIDzv08pn8k6fCUFKSj6Qf8xjk3wAXHgJrknCsFhgBnmdmACOvkA/9wzg0B3iX4TeNIzDk3HLgdCCWYm4AvnHMDgJ8QHE234UpmGcCDwIXOuRLgT8BPnHP7gCuBqWZ2NjAK+Km32oPOuW8ARV584cNq7/He0xPAX4HrvXpl5v1QDsHfFnjAOdcfqAauaxRTN4K/+zDaBQdPXArcbGbdCX77d6BzbjDws2baQg5TSgrS0XzsnFsYtjzWzBYTHD6gP8GdZWN7nHOhYaMXEfzRk0heiFDnNILj8+OcCw0t0lh/YCDwlgV/bWsS3gBtzrml3vovAld5iQJgtJm9R3C4kjO89UNC4/YsA5a54ABx1QR/XCk0gNqnLvj7DRBMQqc1iukUgm3xjhfTOO89bQPqgcfN7HxgdzNtIYep1GQHINJG/k7MzPoCNwPDnXM7zOxPQGaEdfaGzdfR/Oe+Joo6kRiw1Dk3opnnBwEVQKjbKpvgODfFzrkNZvbTRnGH4qgPmw8th+JqfDGw8bIBrznnLm8SrFkpcBZwMTCR4GB9IoDOFKRjywN2ERxpMvRrZ7H2L4K/ooWZFRH5TGQF0MPMhnv10s1soDd/KZADjAQetuA4+lkEd/BbLDji7YUHEFcfM/uGN/9dYG6j598BzvBG5QxdG+nrvV6ec+5vwA+J0B0mhzedKUhHtpjgDvlDYB3BHXisPQQ8ZWYrvNdaQfCo3+ecqzGzi4Dfejv9FODXZvYVwesQI51zG83sMYLXQ642sz9629rE/l/8a4uVwK3eRe9lwNRGMX1pZlcDfw67jfdOYA/wgncdJEDwR19EfLolVaQFFvzRlFTnXLXXXfUG0Nft/+3jZMT0deAvzjl9H0FiTmcKIi3LAWZ5ycGA65KZEETiTWcKIiLi04VmERHxKSmIiIhPSUFERHxKCiIi4lNSEBERn5KCiIj4/j8fObpkxAY/GAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "logistic = LogisticRegression(random_state=0, **best_parameters) \n",
    "train(logistic, X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99       484\n",
      "           1       1.00      0.95      0.97        95\n",
      "\n",
      "   micro avg       0.99      0.99      0.99       579\n",
      "   macro avg       0.99      0.97      0.98       579\n",
      "weighted avg       0.99      0.99      0.99       579\n",
      "\n",
      "Accuracy: 0.9913644214162349\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAF21JREFUeJzt3X20XXV95/H3xwCigqEYsJoEgjxUKT6gKeC4ljIjUmQ0uKwiGdHiUNJaaXW0ts6qo5RWrTrasQOOpkJRVBAc60prKD6hGa1o4oAUUGhEHhJwAIEgAuHB7/yx9yXHS7LvyeXue07C+7XWXZy9z+/s/d0/bs7n7v0757dTVUiStCWPGXUBkqTxZlBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRTapiS5IsnhU7TZK8ldSebMUlm9S3JtkiPax6ck+fSoa9Kjh0GhGdG+kd3TvkH/vyRnJdllpvdTVb9ZVd+Yos31VbVLVT040/tv36Tvb4/zjiT/kuT5M70faZwYFJpJL6+qXYDnAouBd05ukMa2/nv3ufY45wEXAeePuJ4Zl2SHUdeg8bGt/4PVGKqq9cAFwEEASb6R5D1Jvg3cDTwtydwkZyS5Kcn6JH81eKkoyUlJfpjk50muTPLcdv3gJZhDkqxJcmd7FvPhdv2iJDXxZpfkqUlWJLktydokJw3s55Qk5yX5VLuvK5IsHvI4HwA+A8xPssfANl+W5NKBM45nDTy3MMkXktyS5GdJTmvX75vk6+26W5N8Jslu0+n/JMe0+78zyY+THDW57waO/dOT+uzEJNcDX09yQZKTJ237B0le2T5+epKvtP16VZJjp1Ovxp9BoRmXZCFwNHDJwOrXAcuAXYHrgLOAB4D9gIOBI4Hfa1//auAU4PXAE4ElwM82s6uPAB+pqicC+wLnbaGkc4F1wFOBVwHvTfIfBp5f0rbZDVgBnDbkce7U1vgz4PZ23cHAmcDvA08CPg6sSPLYNgj/qT3+RcD8dr8AAd7X1vgMYGHbB1slySHAp4C3t8fzQuDardjEi9r9/zZwDrB0YNsHAnsDX0ryBOArwGeBPYHjgI+2bbSdMSg0k76Y5A7gW8A3gfcOPHdWVV3R/hW+O02QvKWqflFVNwN/Q/NmA01gfKCqVldjbVVdt5n93Q/sl2ReVd1VVRdPbtCG1guAP6uqe6vqUuATNG/wE75VVSvbMY2zgWdPcZzHtsd5D3AS8Kr2uKAJw49X1Xer6sGq+iSwETgMOIQmCN7eHve9VfUtgPYYv1JVG6vqFuDDNG/aW+tE4Mx2W7+sqvVV9aOteP0pbW33AP8APCfJ3u1zrwW+UFUbgZcB11bV31fVA1V1CfC/gVdPo2aNOYNCM+kVVbVbVe1dVX/YvtlMuGHg8d7AjsBN7eWZO2j+8t6zfX4h8OMh9ncicADwoySrk7xsM22eCtxWVT8fWHcdzV/zE3468PhuYOckOyR5bTtofVeSCwbanFdVuwFPBi4Hnjfp2N42cVztsS1s61gIXDcQKg9J8uQk57aX4e4EPk0zBrK1hu27LXno/1PbZ19iU4AvpbnUBs1xHjrpOF8L/Poj2LfGlANWmi2D0xTfQPNX9rzNvWm2z+875Qar/g1Y2g6OvxL4fJInTWp2I7B7kl0HwmIvYP0Q2/8Mm94YN/f8rUmWAWuSfLaqbmprf09VvWdy+/bTUXsl2WEzx/1emj56ZlXdluQVDHkJbJKuvvsF8PiB5c29qU+eTvoc4N1JVgE70wzeT+znm1X1kmnUqG2MZxSade0b6peBDyV5YpLHtIO5E5daPgH8SZLnNR+Syn4Dlz8ekuT4JHtU1S+BO9rVv5y0rxuAfwHel2TndmD5RJq/2GfiWK4CLgT+tF31d8AfJDm0rf0JSf5jkl2B7wE3AX/drt85yQva1+0K3AVsSDKfZoxhOs4A3pDkxW2/zk/y9Pa5S4HjkuzYDti/aojtraQ5eziV5tNeE/37T8ABSV7Xbm/HJL+V5BnTrFtjzKDQqLwe2Am4kmYg+PPAUwCq6nzgPTQDpT8HvkgzrjHZUcAVSe6iGdg+btLlrglLaQaPb6S57v7uqvrqDB7LB4FlSfasqjU04xantce1FjgBoB0DeTnNAP71NAPsr2m38Rc0HyveQHO55wvTKaSqvge8gWbMZwPNWNFEyP43mrON29v9fXaI7W1sazlisH17dnYkzWWpG2ku370feOx06tZ4izcukiR18YxCktTJoJAkdTIoJEmdDApJUqdt7nsU8+bNq0WLFo26DEnapnz/+9+/tar2mLrlw21zQbFo0SLWrFkz6jIkaZuSZHPT4AzFS0+SpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqVNvQZHkzCQ3J7l8C88nyd+muYfxZWnviSxJGi99nlGcRTMN9Ja8FNi//VkG/K8ea5EkTVNvX7irqlVJFnU0OQb4VDXznF+cZLckT2lvarNF994LV189g4VK0hjbfXeYN52b4s6gUX4zez6/eh/lde26hwVFe7vJZQDz5j2NVatmpT5JGqmNG5ugWLp0tHVsE1N4VNVyYDnAAQcsroMPHnFBkjQLrrsObrtt1FWM9lNP64GFA8sLGOKG95Kk2TXKoFgBvL799NNhwIapxickSbOvt0tPSc4BDgfmJVkHvBvYEaCqPgasBI6mufn83TQ3hJckjZk+P/XUOfzSftrpTX3tX5I0M/xmtiSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTr0GRZKjklyVZG2Sd2zm+b2SXJTkkiSXJTm6z3okSVuvt6BIMgc4HXgpcCCwNMmBk5q9Ezivqg4GjgM+2lc9kqTp6fOM4hBgbVVdU1X3AecCx0xqU8AT28dzgRt7rEeSNA079Ljt+cANA8vrgEMntTkF+HKSPwKeAByxuQ0lWQYsA9hzz71mvFBJ0paNejB7KXBWVS0AjgbOTvKwmqpqeVUtrqrFc+fuMetFStKjWZ9BsR5YOLC8oF036ETgPICq+g6wMzCvx5okSVupz6BYDeyfZJ8kO9EMVq+Y1OZ64MUASZ5BExS39FiTJGkr9RYUVfUAcDJwIfBDmk83XZHk1CRL2mZvA05K8gPgHOCEqqq+apIkbb0+B7OpqpXAyknr3jXw+ErgBX3WIEl6ZEY9mC1JGnMGhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqtMOwDZPMB/YefE1VreqjKEnS+BgqKJK8H3gNcCXwYLu6gM6gSHIU8BFgDvCJqvrrzbQ5Fjil3d4Pquo/DVu8JKl/w55RvAL4jaraOOyGk8wBTgdeAqwDVidZUVVXDrTZH/ivwAuq6vYkew5fuiRpNgw7RnENsONWbvsQYG1VXVNV9wHnAsdManMScHpV3Q5QVTdv5T4kST0b9ozibuDSJF8DHjqrqKo/7njNfOCGgeV1wKGT2hwAkOTbNJenTqmqfx6yJknSLBg2KFa0P33sf3/gcGABsCrJM6vqjsFGSZYBywD23HOvHsqQJG3JUEFRVZ9MshPtGQBwVVXdP8XL1gMLB5YXtOsGrQO+227rJ0mupgmO1ZP2vxxYDnDAAYtrmJolSTNjqDGKJIcD/0YzOP1R4OokL5ziZauB/ZPs04bMcTz8rOSLNGcTJJlHE0TXDFu8JKl/w156+hBwZFVdBZDkAOAc4HlbekFVPZDkZOBCmvGHM6vqiiSnAmuqakX73JFJJj52+/aq+tn0D0eSNNOGDYodJ0ICoKquTjLlp6CqaiWwctK6dw08LuCt7Y8kaQwNGxRrknwC+HS7/FpgTT8lSZLGybBB8UbgTcDEx2H/D81YhSRpOzfsp542Ah9ufyRJjyKdQZHkvKo6Nsm/0szF9Cuq6lm9VSZJGgtTnVG8uf3vy/ouRJI0njq/R1FVN7UPbwVuqKrrgMcCzwZu7Lk2SdIYGHZSwFXAzu09Kb4MvA44q6+iJEnjY9igSFXdDbwS+GhVvRr4zf7KkiSNi6GDIsnzab4/8aV23Zx+SpIkjZNhg+ItNDcY+od2Go6nARf1V5YkaVwM+z2KbwLfHFi+hk1fvpMkbcem+h7F/6iqtyT5Rzb/PYolvVUmSRoLU51RnN3+97/3XYgkaTx1BkVVfb99uAa4p6p+CZBkDs33KSRJ27lhB7O/Bjx+YPlxwFdnvhxJ0rgZNih2rqq7Jhbax4/vaC9J2k4MGxS/SPLciYUkzwPu6ackSdI4GfZ+FG8Bzk9yIxDg14HX9FaVJGlsDPs9itVJng78Rrvqqqq6v7+yJEnjYqhLT0keD/wZ8OaquhxYlMSpxyXpUWDYMYq/B+4Dnt8urwf+qpeKJEljZdig2LeqPgDcD9DOJJveqpIkjY1hg+K+JI+jncYjyb7Axt6qkiSNjWE/9fRu4J+BhUk+A7wAOKGvoiRJ42PKoEgS4Ec0Ny06jOaS05ur6taea5MkjYEpg6KqKsnKqnomm25aJEl6lBh2jOL/JvmtXiuRJI2lYccoDgWOT3It8Auay09VVc/qqzBJ0ngYNih+u9cqJElja6o73O0M/AGwH/CvwBlV9cBsFCZJGg9TjVF8ElhMExIvBT7Ue0WSpLEy1aWnA9tPO5HkDOB7/ZckSRonU51RPDRDrJecJOnRaaqgeHaSO9ufnwPPmnic5M6pNp7kqCRXJVmb5B0d7X4nSSVZvLUHIEnqV+elp6qaM90NJ5kDnA68BFgHrE6yoqqunNRuV+DNwHenuy9JUn+G/cLddBwCrK2qa6rqPuBc4JjNtPtL4P3AvT3WIkmapj6DYj5ww8DyunbdQ9r7cC+sqs6pQZIsS7ImyZoNG26Z+UolSVvUZ1B0SvIY4MPA26ZqW1XLq2pxVS2eO3eP/ouTJD2kz6BYDywcWF7QrpuwK3AQ8I12apDDgBUOaEvSeOkzKFYD+yfZJ8lOwHHAioknq2pDVc2rqkVVtQi4GFhSVWt6rEmStJV6C4r2excnAxcCPwTOq6orkpyaZElf+5UkzaxhJwWclqpaCayctO5dW2h7eJ+1SJKmZ2SD2ZKkbYNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE69BkWSo5JclWRtknds5vm3JrkyyWVJvpZk7z7rkSRtvd6CIskc4HTgpcCBwNIkB05qdgmwuKqeBXwe+EBf9UiSpqfPM4pDgLVVdU1V3QecCxwz2KCqLqqqu9vFi4EFPdYjSZqGPoNiPnDDwPK6dt2WnAhcsLknkixLsibJmg0bbpnBEiVJUxmLwewkxwOLgQ9u7vmqWl5Vi6tq8dy5e8xucZL0KLdDj9teDywcWF7QrvsVSY4A/hx4UVVt7LEeSdI09HlGsRrYP8k+SXYCjgNWDDZIcjDwcWBJVd3cYy2SpGnqLSiq6gHgZOBC4IfAeVV1RZJTkyxpm30Q2AU4P8mlSVZsYXOSpBHp89ITVbUSWDlp3bsGHh/R5/4lSY/cWAxmS5LGV69nFJKkR2bjRrj66tHWYFBI0piaOxd++lNYtWomtrbrE6b7SoNCksbUbrvBoYfO1NbmzJnuKx2jkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdeo1KJIcleSqJGuTvGMzzz82yefa57+bZFGf9UiStl5vQZFkDnA68FLgQGBpkgMnNTsRuL2q9gP+Bnh/X/VIkqanzzOKQ4C1VXVNVd0HnAscM6nNMcAn28efB16cJD3WJEnaSjv0uO35wA0Dy+uAQ7fUpqoeSLIBeBJw62CjJMuAZe3S/YsX/9q1vVS8zdk4Fx67YdRVjAf7YhP7YhP7YpM7F0z3lX0GxYypquXAcoAka6puXzziksZC0xd32xfYF4Psi03si02SrJnua/u89LQeWDiwvKBdt9k2SXYA5gI/67EmSdJW6jMoVgP7J9knyU7AccCKSW1WAL/bPn4V8PWqqh5rkiRtpd4uPbVjDicDFwJzgDOr6ookpwJrqmoFcAZwdpK1wG00YTKV5X3VvA2yLzaxLzaxLzaxLzaZdl/EP+AlSV38ZrYkqZNBIUnqNLZB4fQfmwzRF29NcmWSy5J8Lcneo6hzNkzVFwPtfidJJdluPxo5TF8kObb93bgiyWdnu8bZMsS/kb2SXJTkkvbfydGjqLNvSc5McnOSy7fwfJL8bdtPlyV57lAbrqqx+6EZ/P4x8DRgJ+AHwIGT2vwh8LH28XHA50Zd9wj74t8Dj28fv/HR3Bdtu12BVcDFwOJR1z3C34v9gUuAX2uX9xx13SPsi+XAG9vHBwLXjrrunvrihcBzgcu38PzRwAVAgMOA7w6z3XE9o3D6j02m7Iuquqiq7m4XL6b5zsr2aJjfC4C/pJk37N7ZLG6WDdMXJwGnV9XtAFV18yzXOFuG6YsCntg+ngvcOIv1zZqqWkXzCdItOQb4VDUuBnZL8pSptjuuQbG56T/mb6lNVT0ATEz/sb0Zpi8GnUjzF8P2aMq+aE+lF1bVl2azsBEY5vfiAOCAJN9OcnGSo2atutk1TF+cAhyfZB2wEvij2Slt7Gzt+wmwjUzhoeEkOR5YDLxo1LWMQpLHAB8GThhxKeNiB5rLT4fTnGWuSvLMqrpjpFWNxlLgrKr6UJLn03x/66Cq+uWoC9sWjOsZhdN/bDJMX5DkCODPgSVVtXGWapttU/XFrsBBwDeSXEtzDXbFdjqgPczvxTpgRVXdX1U/Aa6mCY7tzTB9cSJwHkBVfQfYGZg3K9WNl6HeTyYb16Bw+o9NpuyLJAcDH6cJie31OjRM0RdVtaGq5lXVoqpaRDNes6Sqpj0Z2hgb5t/IF2nOJkgyj+ZS1DWzWeQsGaYvrgdeDJDkGTRBccusVjkeVgCvbz/9dBiwoapumupFY3npqfqb/mObM2RffBDYBTi/Hc+/vqqWjKzongzZF48KQ/bFhcCRSa4EHgTeXlXb3Vn3kH3xNuDvkvwXmoHtE7bHPyyTnEPzx8G8djzm3cCOAFX1MZrxmaOBtcDdwBuG2u522FeSpBk0rpeeJEljwqCQJHUyKCRJnQwKSVIng0KS1MmgkCZJ8mCSS5NcnuQfk+w2w9s/Iclp7eNTkvzJTG5fmmkGhfRw91TVc6rqIJrv6Lxp1AVJo2RQSN2+w8CkaUnenmR1O5f/Xwysf3277gdJzm7Xvby9V8olSb6a5MkjqF96xMbym9nSOEgyh2bahzPa5SNp5ko6hGY+/xVJXkgzx9g7gX9XVbcm2b3dxLeAw6qqkvwe8Kc03xCWtikGhfRwj0tyKc2ZxA+Br7Trj2x/LmmXd6EJjmcD51fVrQBVNXE/gAXA59r5/ncCfjI75Uszy0tP0sPdU1XPAfamOXOYGKMI8L52/OI5VbVfVZ3RsZ3/CZxWVc8Efp9mIjppm2NQSFvQ3jXwj4G3tVPZXwj85yS7ACSZn2RP4OvAq5M8qV0/celpLpumcP5dpG2Ul56kDlV1SZLLgKVVdXY7RfV32ll67wKOb2cqfQ/wzSQP0lyaOoHmrmrnJ7mdJkz2GcUxSI+Us8dKkjp56UmS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmd/j+6jfos6eS6oAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluate(logistic, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Naive Bayes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## No hyperparameters to tune"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XmYFOW1+PHv6ZmefQYEAdkhCsoOwwQDioKowfwSSdBEDSbuY4wYDdEblFw1RgjxGq8x8ap4o9E4ytUY1CTuCBpiXEARBEQQ2ZFl0IHZezm/P6q6prunZwGne2aY83meerr67beqT1d3v6fqreq3RVUxxhhjAHytHYAxxpi2w5KCMcYYjyUFY4wxHksKxhhjPJYUjDHGeCwpGGOM8VhSMMYY47GkYIwxxpO0pCAiD4nIHhH5sIHHRUTuEZGNIrJKRAqTFYsxxpjmSU/iuv8E/AF4tIHHzwIGudOJwH3ubaOOPvpoHTBgQMtE2ISKigpyc3NT8lwtyeJOnfYYM1jcqdRWYl6xYsU+Ve3WZEVVTdoEDAA+bOCxB4ALou6vB3o2tc6xY8dqqixZsiRlz9WSLO7UaY8xq1rcqdRWYgaWazPa7dY8p9Ab2BZ1f7tbZowxppUks/uoxYhIMVAM0KNHD5YuXZqS5y0vL0/Zc7WkpuLu/uqrfOV//5fMPXuo6d6dTZdfzp7TT09dgA3EUXDhhSxt5RgOdVu01Gck1e9JQ3G31c9GJI5UfydbYnu0RMwpfV+aczhxuBPWfRTrscdU+/dXFXFuH3usZdfvajTuxx5TzclRhbopJydpsRxKHMHMzNTG0dC2ePRR1WBQNRBQra11ppoa1epqZ6qqUq2sVK2s1NdfeEG1vNyZDh5UPXDAmcrKVL/4wpk+/9yZ9u9XLS11pn37VPfudab77lPNzo6NIztb9Q9/UN25U3XHDmfavl112zZn2rrVmbZsUd282Zk+/dSZNm1S/eQTZ9q40Zk2bFD9+GNnWr9e33r0UdWPPnKmdetU165VveMO1ays2DiyslTvvNNZx6ZNzvNs3erEsGOH6q5dqrt3q+7Z47ym/fud13zggLM9Kiqc7VVT42zHYFA1HD689+Wxx1LbFdNC35UvHXMLxUEzu49Ekzh0togMAP6uqsMTPPb/gJnAN3BOMN+jquOaWmdRUZEuX768hSNNbOnSpUyaNKllVlZSAsXFUFlZV5aTAwsWwIwZzVuHKtTWQnk5VFTUTdH3y8vZsHIlg3r1Slzv5Zehurr+ujMyYMwYSEuD9PTYW5+vflmiepH56Prx9Xw+Z37uXPj88/pxdO4MP/0pBALOFAw6rzkYrLsfmY/cb2w+un70fCgEZWXONjWtw+cDkfq31dWJ3xefj0BeHv70dKeeiFMefRuZ9/lilqv3eGQ+0bKR+c2bnc9LvPR0OPbYuvrREpSVV1eTl53dZL0Gyz76yPnMxuvf34mxmURkhaoWNVUvad1HIvIEMAk4WkS2A7cAfgBVvR94HichbAQqgUuSFUubMGdObEIA5/7VV8PKlbGNe3l5bINeWVl3PxRq8qkGRWZ8PifxZGc7tzk5iRMCOA1vRobzJaiudp7nUKdwuH7ZoTa6X3wBt9xSF38kmcRP0eWReb+/bj4rC/LyEi/n9zvzjz/ecBwzZzq3DTUc7v1PvviCY486qvHGJTLf0OO3395wHJFtIeJsy0jjGb3OxhrABuJZu2cPQ485Jna5a6+lZATMmQJbO0G/Mpi7GGasBu6803l/oyfVuvc8Mt9Qnch8/P1w2Hn+6LI//SnxtgiH2XPKKfTOy6v7XEWWj6w7Mp+oLNEy0bfR9VRh48YGtkcQ+vVr8C2LoUpVVVVsUkj0nYgrc3bW3bJAgMcTxfHh1ubFcIiSeqSQDG3iSCH2QK7uwx19v7TUaexXroT334dFixp+oqysukY7vhGPzOfmxpZHT7m5TgOYnw8FBfzr0085afx457HI3nlkOu442Jrgw9SvH2za1PAXKfq1JXos8qWG+l/6QMBJOuFw3V7/1Knw2Wf14+jZE15/va4Bj24EoW7+UG+hfgM9ZAhsi77WwdW3L3z8cf3yBJb++99MGj8+8YOJ9vwSGTyYkk5b63/py/rBxo2JY2+qjEjDAorbNRB1+69//osJJ0+IKfvrmX25csJ+KjPqVpdTCw+82YVvv7Sl/ssj7vWp1pVEfT5Eqfe5kUiSiyoXdzZjxGgeP2p7ve3x/c/7sPRPD3PymNFORZ/PWU90PNH347ZPTLwiMcsmeq9KzupD8Un1t8eCf3Xhghe21dumDd2+8+5KCscOJxQOESZMOBwmjLq3YcLqTOrehsNhokN96/uncNXJZfXjeLMrM5bsqxd3Q1r9SKFNCoWcBim+AY9u1KL3egMBp9GIfizS8EUrLYU1a2DtWud2zRrYsaPu8V69nIY/0V56nz6wbl1swx3Zo4uePwSBsjKncU1k3rzE3Vjz5jmNcKrceScl/30JcyYGvC/+r15P4wc/+y8YNKjp5VvCr39dL4a5//Qz46e/dt6v5hBpsm5jjQbAwp9P5cc7F1Dpd+pv6QzFZ0NtzzOYHqqkNlRLVaCKqmAV1cFqaoI1VAerqQpWURWoojpUTU2ghpqQU14dcurUhGqoDdVSE3Rua0O11IRqqAnWsH/ffvzb/XXlwRo+mFRGfCdFZQZcOqmM3z40kXRfujOlpeP3+evu++ru+9P8MWVpvjSvLFInTdKc+fj1uPdXXns895dtpya9bntcNg2WF5xAzy/eY+vWHYQ17DSyGiakIVSVoAa9BjakITSshNSpEwwHUdR7LBwOxzXS4Zh1hcNhSk6roFLrb4/i08p55m8XxT6XKmGcdarWPW9Yw1QcqCBrS5ZzP+75nGSg3jq8dUXi1DDbTjtIfP9AZQbMOR2a2fF8SDpWUigthf37Eze08YfbkT2ZcNiZj3Q/7N0Lq1fHTrt21a1nwAAoLIQf/hBOOAEGD4Zu3eCVV+BnP6vfGM+f7+zlp0rk/MWcOc4RQ79+Tv9+c89rtJCSkVB8tnhfOqchFHwjD/+DHn3Uq+6hd6QsFA5RG64lGA4SCAWoDdXy1LF7ufFsqI6K4bKzYe0xazll40sEQgFqQjUEwgFvmUhDWxuqJRAKsGXrFl569SWnLOyUBUIBAuGAVyeoQYKhoLOecMCLIRgOEgwHWVu6loA/9rVU+uGyfQ9xxR1/IqRNdxk2JU3SyEzPJCMtg6y0LCQk5IXyvLLMtEwC9ZoeR4AQBVkFXsw1tc42ibymYLjutjbkbOPIY4cde1zLVJMOd1e+Ch++CgnHSGgZguATH2m+NGq1NmGdSq1l2bZlTj1Jwye+mClN0hAR0nzOYzXBGsLBsPdYmi8Nv/gT18eHzxe7rs1fbE4Yx9bg/qRsg46VFFSdfvPm7AWqkrl3r3OkEJ0A9u51HhdxTjZ97WswYoQzDR3qrDsQcPbyO3VyunSysuBHP3LmW7kxBigZATddp2wrg76dlF8ND3F+qNZ92XVdDs25H10W1nDMbXWwmoM1BzlQc4CDtQedqeYg5bXlzF48m8q4L121BLnyb1fy9/V/r9d4Ru7Xm49qXOMfi260ouNtTI0GmLdsHvOWzWv+Bv3UuRGEjLQMb883Mu9P8+P3+WP2jv1pfjLTM8nz5REIJziJ6G7jH3/1x2SmZ5KZ5k7pmWT4Mpzb9AwyfBlkpNVNmemZ+H1+MtMy8af5vfJ0X3rM0cmmlZsYOGogSqQrByY/OpmdB3fWi6NXXi8e/OaDTpeGAlLXFSMiTc4Hw0FCGiIQChAKh5wkmeC9jdw/58lzEr5fgvCbEb/h2CHH1jWkbiPrNcqRRtW9jTTyPvF5MXmNr9TVERGvLPI6G9we+b1YctESUGK2nxukF2tkO3zy/iccN+a4utcR150Vv93iH3tnxzvsOBjV8+Dq16mZ5zUOUcdICiUldY1xz55w440wfXrd46qwZYvT6H/4IaxaBatXMz5ydYzP5+zxT5pUlwCGDXP68gFqapy+chGn3/+YY5xEEN2fDU4CaKEkEDnEDGvYOyyOnj6v+tz5MoZDhDTkzT+z/hnmvDaH6qDTlbW1bCtX/v1K9lbs5VuDv0UgHKAiUEF5TblzGyinojbqfq1zW1FbN19eWx4zH3msocauMRWBCl7b/Fpdd0SkCyK6QfX5yU7Prut6SEtP2JURPaVJWl1dccpue+O2BuP407Q/xTTk0ev2ukPS/Oz4cAfHjT7OW3+99ynSaCR8E52byY9MZmd54sZ45riZXkMcWcbnfq581DV0InWNHxCz5xp9P1J3m28bvQp6AXhlt0++naufv5qqYJUXQ3Z6Nrefdju9C3p7ryf6swfU++zFl0ee15fuxOL1mbuvS1Xd8wFOWc/8ngkb4575PRlZMJKB3QfWbZO4hjmSECLbPH4vPtH2iS6P3p63T76dmc/PpDJYd3Sfk57D7ZNvp29B35gkG7/TFL0NtsgWOmd1rrfTFL+tom8j3UgAs8bPivnOAuT4c5g7ZW69bdQSjvykEH8p6M6dcP31sHy503BHEsGBA87jfj8cfzxMncrHXbsy+Otfd05Ixl9SFrk0VNVJDt26OXUOoV8+8uWKb9Qj5ZE93uh5rx/U/fB4X6jIOlEC4QClVaUEQs5taWUp+yr3sa9yH3P/OTfmwwXOHv1/vPof/OK1X1AdauDqpDjZ6dnkZ+aT68/1brvmdCUvI4+8jDzyM/LJzcglPyOfHH8OuRm55PpzyU7PJjcjl4ufuZjdFbvrrbdXnrsXFt9YOC8upkGM7LFG7wXGf+kbm3945cNsO1D/RHPfgr58d9h3E77u+JOrFRkVHNfF2QtsqPGvd0I2+jER5p0+jx//48dUBqIaH38O80+fz7FHHRuz99hggjlEPvGRlxHbbXnJmEvISM9gzuI5bC3bSr9O/Zg7ZS4zRrT80Wz8eZXocy3zTnO3R1xjPO+0eWR8kcHAzgNjGm9o2W0T0VLbI82XRtecrocdx6zxs+iR1yMl7wt0hKSQ6FLQmhp45BHIzHQa/GnT6o4Ajj/eKQd2rlnD4GHD6paLXK6pWndEELnCpxlKVpdw46s3sv3Adnrm92TW12bxzcHfrNeoR+/lxO8FpvvSqQhUsLdyL3sq9iSctu3bRtnyMj6vTvA7gAaENcwlYy5J2KhHbvMy88jz55Hjz8Envnp7h5EvdfwVIZFD/OgulbmnzeWaF66J2SvN9GUy//T5fKXLVxo9tG4pvz791xT/rbheY/zr039Njj+nWesQEbL92U1XbMRFoy4i3Zeesi99Y2aMmJGS5/Xe1wRv6UWjLyI9LfH2WLp0Kf40f/2FkiRV26MtxXHkJ4VEl1+C09Wzfr1zZNCYUMhJBOGwkyx69HASQVPLxXlk5SNc9Y+rvEZw58Gd/OeS/yQzPZPpQ6YTDAfZV7kvYSO/t2Ivuyt2O4mgfE/CvfmMtAy653anW043emX3YmLvifTI7UG33G50z+3uTd9e+O2E/ZO983sz++TZ9a6IiIg+IqkN1cb0m0e6VCJ9u/En3xI15pcVXkaWPyvmi39hzwv5wagfHNJ2/TIiX7KO1Bi3F7Y9Ws+RnxT69XPOF8Tr1avhhj0crksEgQAcfbTTRZSRkbh+IwKhAKWVpdy0+KaYvWKAqmAVP33pp/zy9V9SWlma8ORa58zOdM9zGvuinkX1GvnI1Cmzk9f4rnl3DcO+WneEE2ngg+Fgwv7JrPQsZo2fRSgcijlRGunP966EiJpvCfFf/NYYZ8oaH2NiHflJYe7c+tflZ2fD7Nmx9cJhp1spGHS6gzp3dpLAwIGH/DsBcC6B/Lzqc0qrSknzpbGrfFfCesFwkK8f+/WEDX23nG5kpmc2+Vxeox8Kej9+Ka8t9/bufT6fd3L2ktGX0DmrM7cuvZXtB7bTt1Nf5p02jxkjrWE0xnSEpBB/XX701UeqzhFBMFj/EtL4IQKaKaxhDtYcZE/FHgDyMvLYsH8D6b70hFfj9M7vzR1n3NHoOlXVu6wv0ujHX3Hh9/nJSc8hMz2TrWlb6VvQ1+vHj9+zv3TMpVw65tJDel3GmI7hyE8KUHcp6O7dzvhBInDwoJMI3KEhEl5CeghUlfLacvZW7CUQDngnYx9e+TBz35hLRloGIkJtqO7a/Oz0bGafPPuQG31/mj/m5G18o+8T35c++WmM6Zg6RlKIEHEuJc3Ph+7dnW6kL5EIIqoCVeyp2EN1sJpsfzZZ/ix2l+/mZy//jCWbl3DawNP47Zm/ZdnWZcxfNp+dB3fSM68ns8bP4syvnElVsIqMtAxy03OdE7duo5/uS/f68Y0xJhU6VlLo0gW6dm2xMX5qgjXsq9zHwdqDZKZlkp+ZD8ALG17ghlduoCpQxdzT5nLRqIsQEb59wrc5feDp3vX81ugbY9qajpUUmvl7gqYEw0H2V+1nf9V+/D4/BZkFAFTUVnDL0lt44sMnGNF9BL8/6/cM6jrIW6aytpKe+T3plNWpReIwxpiW1rGSwpcUCocoqyljX8U+RIT8jHzvMtAVO1fwkxd+wpayLcwcN5Ofjf8ZGWnOJay1oVpqg7X069yv2T+KMsaY1mBJoRlU1buiKKxhcjJyvC6fYDjIPW/fw91v3U3P/J48/b2nObHPid6yVQHntwn9O/dv1uWlxhjTmiwpNKGitoLdFbsJhJwritJ8decjPv38U6554Rre/+x9pg+ZztzT5npdSZFlM9Iy6F3QO+FgacYY09ZYS9WA6mA1taFath3YRnZ6NlmZdcNtqyoLP1zIzUtvxu/z8z/f+B+mnTAt5vGDtQcpyCjgmPxj7ESyMabdsKQQpzZUS2llKWXVZQAxe/4A+6v2c8PLN/DiJy8yoe8E7p56N73ze3uPR368dnTO0Rydc3SLD+JmjDHJZEnBFQwH+aLqC0qrSkn3pVOQVVCvQV/y6RJmvTyLL6q/4D9P+U+KxxbHHAVErjDqld/LrjAyxrRLHT4phDVMWXUZ+yr3IQh5GXn1kkFVoIp5/5zHQysf4viux/PY9McY1m1YTJ3I/9z27dSX3IzcVL4EY4xpMR02KUSGpdhTsYeQhrxhKeJ9uOdDZj4/kw37N3DZmMu4aeJNZKXH/p1n5AqjAZ0H2BVGxph2rUMmhcpApfO/BMFqcjJyyPbVHycoFA7x5LYneeRfj9AluwuPT3+cUwecWq9e5AqjXvm9UvrnH8YYkwwdKinUhmrZU76H8tpysvxZFGQVJKy348AOrn3xWv69/d9847hv8JszfkOX7C4xdSJHGnkZeRyTd0zMparGGNNedaik8HnV51QFqxpMBgCL1i3iptduIhQOcf3g67nu/11X7xxDWJ3/K+ia3dWuMDLGHFE6VFIAGtyjL6suY85rc1j00SKKehVxz9R7KN9QXq/BD4VDVNRWcEz+MXTO6pyKkI0xJmU6XFJI5M1tb3Lti9eyu3w3N0y4gZnjZpLuS2cNa2Lq2RVGxpgjXYdOCjXBGu58807uW34fAzoP4Nnzn2VMzzEJ60aPYRR/9ZExxhwpOmxS+Lj0Y2Y+P5M1e9cwY8QMbjn1lgb3/itrK0n3pdO7oLddYWSMOaJ1iKRQsrqEOYvnsLVsKz3zezKhzwT+/vHfycnI4eFpD3PmsWc2uOzBmoN2hZExpsM44pNCyeoSiv9WTGWgEoCdB3fyl3V/YejRQyk5p4Tuud0TLhfWMKFwiC7ZXewKI2NMh3HEJ4U5i+d4CSFaWU1ZgwkhcoWRP81Pt9xuyQ7RGGPajCN+TOetZVsTlu88uDNheW2olqpgFX0K+pAm1l1kjOlYjvik0K9Tv4TlvfJ71SurDlYTDAXp16kfeZl5yQ7NGGPanCM+KcydMrfe/yJnp2cz++TZMWWVtZX48Nklp8aYDu2ITwozRsxgwbcW0L9TfwShV34v7jjjDqYPmQ7U/f9ytj+bvp362iWnxpgO7Yg/0QxOYpgxYga7y3dTEajwjgQiYxh1ye5Ct5xudoWRMabD6xBJIZFQOER5bTnH5B3DUdlHtXY4xhjTJnTIpBAIBagJ1dC3oK+dUDbGmChJPacgIlNFZL2IbBSR2Qke7yciS0TkfRFZJSLfSGY84Ix3FAgF7AojY4xJIGlJQUTSgHuBs4ChwAUiMjSu2i+AJ1V1DHA+8D/JiiciOz3brjAyxpgGJPNIYRywUVU3qWotsBCYFldHgcg/3nQCEv+irIUclX2UXWFkjDGNSOY5hd7Atqj724ET4+rcCrwsItcAucDpSYyHjLSMZK7eGGPaPVHV5KxY5Fxgqqpe7t7/AXCiqs6MqjPLjeG3IjIe+CMwXFXDcesqBooBevToMXbhwoVJiTleeXk5eXnt77yDxZ067TFmsLhTqa3EPHny5BWqWtRkRVVNygSMB16Kun8jcGNcnTVA36j7m4Duja137NixmipLlixJ2XO1JIs7ddpjzKoWdyq1lZiB5dqMtjuZ5xTeBQaJyEARycA5kfxcXJ2twBQAERkCZAF7kxiTMcaYRiQtKahqEJgJvASsw7nKaI2I3CYiZ7vVfgZcISIfAE8AF7sZzRhjTCtI6o/XVPV54Pm4spuj5tcCJyUzBmOMMc13xA+IZ4wxpvksKRhjjPFYUjDGGOOxpGCMMcZjScEYY4zHkoIxxhiPJQVjjDEeSwrGGGM8lhSMMcZ4LCkYY4zxWFIwxhjjsaRgjDHGY0nBGGOMx5KCMcYYjyUFY4wxHksKxhhjPJYUjDHGeCwpGGOM8VhSMMYY47GkYIwxxmNJwRhjjMeSgjHGGI8lBWOMMR5LCsYYYzyWFIwxxngsKRhjjPFYUjDGGOOxpGCMMcZjScEYY4zHkoIxxhiPJQVjjDEeSwrGGGM8lhSMMcZ4LCkYY4zxWFIwxhjjsaRgjDHGY0nBGGOMx5KCMcYYjyUFY4wxnqQmBRGZKiLrRWSjiMxuoM73RGStiKwRkceTGY8xxpjGpSdrxSKSBtwLnAFsB94VkedUdW1UnUHAjcBJqvq5iHRPVjzGGGOa1uwjBRE5WUQucee7icjAJhYZB2xU1U2qWgssBKbF1bkCuFdVPwdQ1T3ND90YY0xLE1VtupLILUARcLyqDhaRXsBTqnpSI8ucC0xV1cvd+z8ATlTVmVF1ngE+Bk4C0oBbVfXFBOsqBooBevToMXbhwoWH8BIPX3l5OXl5eSl5rpZkcadOe4wZLO5UaisxT548eYWqFjVVr7ndR98BxgDvAajqThHJ/xLxRT//IGAS0Ad4Q0RGqOoX0ZVUdQGwAKCoqEgnTZrUAk/dtKVLl5Kq52pJFnfqtMeYweJOpfYWc3O7j2rVOaRQABHJbcYyO4C+Uff7uGXRtgPPqWpAVT/FOWoY1MyYjDHGtLDmJoUnReQBoLOIXAG8CjzYxDLvAoNEZKCIZADnA8/F1XkG5ygBETkaGAxsamZMxhhjWlizuo9U9U4ROQM4ABwP3KyqrzSxTFBEZgIv4ZwveEhV14jIbcByVX3OfexMEVkLhIAbVLX0S7weY4wxX0KTScG9tPRVVZ0MNJoI4qnq88DzcWU3R80rMMudjDHGtLImu49UNQSERaRTCuIxxhjTipp79VE5sFpEXgEqIoWq+pOkRGWMMaZVNDcp/NWdjDHGHMGae6L5EfcKosFu0XpVDSQvLGOMMa2hWUlBRCYBjwCbAQH6ishFqvpG8kIzxhiTas3tPvotcKaqrgcQkcHAE8DYZAVmjDEm9Zr74zV/JCEAqOrHgD85IRljjGktzT1SWC4i/ws85t6fASxPTkjGGGNaS3OTwlXA1UDkEtR/Av+TlIiMMca0muYmhXTgd6p6F3i/cs5MWlTGGGNaRXPPKSwGsqPuZ+MMimeMMeYI0tykkKWq5ZE77nxOckIyxhjTWpqbFCpEpDByR0SKgKrkhGSMMaa1NPecwnXAUyKy073fEzgvOSEZY4xpLY0eKYjIV0XkGFV9FzgB+D8gALwIfJqC+IwxxqRQU91HDwC17vx44CbgXuBz3P9MNsYYc+RoqvsoTVX3u/PnAQtU9WngaRFZmdzQjDHGpFpTRwppIhJJHFOA16Iea+75CGOMMe1EUw37E8DrIrIP52qjfwKIyHFAWZJjM8YYk2KNJgVVnSsii3GuNnrZ/U9lcI4wrkl2cMYYY1KryS4gVX0rQdnHyQnHGGNMa2ruj9eMMcZ0AJYUjDHGeCwpGGOM8VhSMMYY47GkYIwxxmNJwRhjjMeSgjHGGI8lBWOMMR5LCsYYYzyWFIwxxngsKRhjjPFYUjDGGOOxpGCMMcZjScEYY4zHkoIxxhiPJQVjjDEeSwrGGGM8SU0KIjJVRNaLyEYRmd1IvXNEREWkKJnxGGOMaVzSkoKIpAH3AmcBQ4ELRGRognr5wLXA28mKxRhjTPMk80hhHLBRVTepai2wEJiWoN6vgN8A1UmMxRhjTDMkMyn0BrZF3d/ulnlEpBDoq6r/SGIcxhhjmim9tZ5YRHzAXcDFzahbDBQD9OjRg6VLlyY1tojy8vKUPVdLsrhTpz3GDBZ3KrW7mFU1KRMwHngp6v6NwI1R9zsB+4DN7lQN7ASKGlvv2LFjNVWWLFmSsudqSRZ36rTHmFUt7lRqKzEDy7UZbXcyu4/eBQaJyEARyQDOB56LSkZlqnq0qg5Q1QHAW8DZqro8iTEZY4xpRNKSgqoGgZnAS8A64ElVXSMit4nI2cl6XmOMMYcvqecUVPV54Pm4spsbqDspmbEYY4xpmv2i2RhjjMeSgjHGGI8lBWOMMR5LCsYYYzyWFIwxxngsKRhjjPFYUjDGGOOxpGCMMcZjScEYY4zHkoIxxhiPJQVjjDEeSwrGGGM8lhSMMcZ4LCkYY4zxWFIwxhjjsaRgjDHGY0nBGGOMx5KCMcYYjyUFY4wxHksKxhhjPJYUjDHGeCwpGGOM8VhSMMYY47GkYIwxxmNJwRhjjMeSgjHGGI8lBWOMMR5LCsYYYzyWFIwxxnjSWzuAlhAIBNi+fTvV1dUtut5OnTqxbt26Fl1nKrT3uLOysujTpw9+v7+1QzKmwzkiksL27dvJz89nwIABiEiLrffgwYPk5+e32PpSpT3HnZeXR2lpKdu3b2fgwIGtHZIxHc4R0X0yCu3/AAAVdElEQVRUXV1N165dWzQhmNYhInTt2rXFj/qMMc1zRCQFwBLCEcTeS2NazxGTFFpTaWkpo0ePZvTo0RxzzDH07t3bu19bW9usdVxyySWsX7++0Tr33nsvJSUlLRGyMcYkdEScUzhkJSUwZw5s3Qr9+sHcuTBjxmGvrmvXrqxcuRKAW2+9lby8PK6//vqYOqqKquLzJc7DDz/8cJPPc/XVVx92jMnU1GszxrQfHe9bXFICxcWwZQuoOrfFxU55C9u4cSNDhw5lxowZDBs2jF27dlFcXExRURHDhg3jtttu8+qefPLJrFy5kmAwSOfOnZk9ezajRo1i/Pjx7NmzB4Bf/OIX3H333V792bNnM27cOI4//njefPNNACoqKrjwwgsZOnQo5557LkVFRV7CinbDDTcwdOhQRo4cyc9//nMAPvvsM6ZNm8bIkSMZNWoUb7/9NgB33HEHw4cPZ/jw4fz+979v8LW98MILjB8/nsLCQs477zwqKipafJsaY5LryDtSuO46SNAIet56C2pqYssqK+Gyy+DBB2OKs0MhSEuD0aPBbYwP1UcffcSjjz5KUVERAPPnz6dLly4Eg0EmT57Mueeey9ChQ2OWKSsr49RTT2X+/PnMmjWLhx56iNmzZ9dbt6ryzjvv8Nxzz3Hbbbfx4osv8vvf/57u3bvz7LPP8sEHH1BYWFhvud27d/P888+zZs0aRIQvvvgCcI5EzjjjDGbOnEkwGKSyspK3336bkpIS3n33XYLBIOPGjWPSpElkZ2fHvLY9e/Ywf/58Fi9eTE5ODnPnzuV3v/sdN91002FtN2NM6+h4RwrxCaGp8i/p2GOP9RICwBNPPEFhYSGFhYWsW7eOtWvX1lsmOzubs846C4CxY8eyefPmhOuePn16vTrLli3j3HPPBWDUqFEMGzas3nJdunTB5/NxxRVXsGjRInJzcwFYunQpV155JQDp6ekUFBSwbNkyzjnnHLKzs8nPz+fb3/42//znP+u9tjfffJO1a9cyYcIERo8eTUlJSYNxG2PariPvSKGpPfoBA5wuo3j9+8PSpTFFVS1wvX+kwQXYsGEDv/vd73jnnXfo3LkzF154YcJLLzMyMrz5tLQ0gsFgwnVnZmY2WScRv9/P8uXLeeWVV3jqqae47777ePnll4FDu/In+rWpKlOnTuXPf/5zs5c3xrQ9ST1SEJGpIrJeRDaKSL3+DxGZJSJrRWSViCwWkf7JjAdwTirn5MSW5eQ45Ul24MAB8vPzKSgoYNeuXbz00kst/hwnnXQSf/3rXwFYvXp1wiORgwcPcuDAAb75zW/y3//937z//vsATJ48mfvvvx+AUCjEgQMHmDhxIosWLaKqqory8nKeffZZJk6cWG+dEyZM4PXXX2fTpk2Ac25jw4YNLf76jDHJlbQjBRFJA+4FzgC2A++KyHOqGt1KvQ8UqWqliFwF3AGcl6yYgLqrjFrw6qPmKiwsZOjQoZxwwgn079+fk046qcWf45prruH73/8+Q4cO9aZOnTrF1CkrK2P69OnU1NQQDoe56667APjDH/7AFVdcwQMPPEB6ejoPPPAA48aN44ILLuCrX/0qAFdddRUjRoxg48aNMevs0aMHf/zjHznvvPO8y3DnzZvHoEGDWvw1GmOSKHI5YUtPwHjgpaj7NwI3NlJ/DPCvptY7duxYjbd27dp6ZS3hwIEDSVlvMgUCAd2zZ4+qqn788cc6YMAADQQCrRxV80Rv72S9py1tyZIlrR3CYbG4U6etxAws12a03ck8p9Ab2BZ1fztwYiP1LwNeSGI8HUJ5eTlnnnkm4XAYVfX2+o0xpjnaRGshIhcCRcCpDTxeDBSD002xNO6EcKdOnTh48GCLxxUKhZKy3mRKS0tjyZIlpKWleWXt5TVEb+/q6up673NbVF5e3i7ijGdxp057izmZSWEH0Dfqfh+3LIaInA7MAU5V1YTXharqAmABQFFRkU6aNCnm8XXr1iVlVND2PNpoe487KyuLMWPGtHJETVu6dCnxn8f2wOJOnfYWczKvPnoXGCQiA0UkAzgfeC66goiMAR4AzlbVPUmMxRhjTDMkLSmoahCYCbwErAOeVNU1InKbiJztVvsvIA94SkRWishzDazOGGNMCiT1nIKqPg88H1d2c9T86cl8fmOMMYem4w1zkQSTJ0+u90O0u+++m6uuuqrR5fLy8gDYuXOnNzRFvEmTJrF8+fJG13P33XdTWVnp3T/nnHO88YyMMeZQdMikULK6hAF3D8D3Sx8D7h5AyeovN0LqBRdcwMKFC2PKFi5cyAUXXNCs5Xv16sVf/vKXw37++KTw9NNP07lz58NeX7KEQqHWDsEY04QOlxRKVpdQ/LditpRtQVG2lG2h+G/FXyoxnHvuufzjH//wfsm7efNmdu7cycSJEykvL2fKlCkUFhYyYsQInn322XrLb968meHDhwNQVVXF+eefz5AhQ/jOd75DVVWVV++qq67yht2+5ZZbALjnnnvYuXMnkydPZvLkyQAMHz6cffv2AXDXXXd5w15Hht3evHkzQ4YM4YorrmDYsGGceeaZMc8T8dRTTzF8+HBGjRrFKaecAjgN+/XXX8/w4cMZOXKkN5T24sWLGTNmDCNGjODSSy+lxh1gcMCAAfz85z+nsLCQp556ik8++YSpU6cyduxYJk6cyEcffXTY290Y0/LaxO8UWtJ1L17Hys8aHjr7re1vUROKvfK1MlDJZc9exoMrYofODoVCpKWlMfqY0dw9teGB9rp06cK4ceN44YUXmDZtGgsXLuR73/seIkJWVhaLFi2ioKCAffv28bWvfY2zzz67wYHn7rvvPnJycli3bh2rVq2KGfp67ty5dOnShVAoxJQpU1i1ahU/+clPuOuuu1iyZAlHH310zLpWrFjBww8/zNtvv42qcuKJJ3Lqqady1FFHsWHDBp544gkefPBBvve97/H0009z4YUXxix/22238dJLL9G7d2+vO2rBggVs3ryZlStXkp6ezv79+6murubiiy9m8eLFDB48mB/+8Ifcd999XHfddYDzJ0TvvfceAFOmTOH+++9n0KBBvP322/z4xz/mtddea3DbGmNSq8MdKcQnhKbKmyu6Cym660hVuemmmxg5ciSnn346O3bsYPfu3Q2u54033vAa55EjRzJy5EjvsSeffJLCwkLGjBnDmjVrEg52F23ZsmV85zvfITc3l7y8PKZPn+4Nez1w4EBGjx4NNDw890knncTFF1/Mgw8+6HX9vPrqq1x55ZXer6S7dOnC+vXrGThwIIMHDwbgoosu4o033vDWc955znBW5eXlvPnmm3z3u99l9OjRXHnllezatavR12CMSa0j7kihsT16gAF3D2BLWf2hs/t36s/Si5fGlB3Kj8CmTZvGT3/6U9577z0qKysZO3YsACUlJezdu5cVK1bg9/sZMGBAwuGym/Lpp59y55138u6773LUUUdx8cUXH9Z6IiLDboPzK+hE3Uf3338/b7/9Nv/4xz8YO3YsK1asOKznigyxHQ6H6dy5c8J/gjPGtA0d7khh7pS55Phjh87O8ecwd8qXGzo7Ly+PyZMnc+mll8acYC4rK6N79+74/X6WLFnClkT/5RDllFNO4fHHHwfgww8/ZNWqVYAz7HZubi6dOnVi9+7dvPBC3TBR+fn5CYeymDhxIs888wyVlZVUVFSwaNGihMNeN+STTz7hxBNP5LbbbqNbt25s27aNM844gwceeMD7/4b9+/dz/PHHs3nzZm/k1D//+c+cemr9EUsKCgoYOHAgTz31FOAcRX3wwQfNjscYk3wdLinMGDGDBd9aQP9O/RGE/p36s+BbC5gx4ssPnX3BBRfwwQcfxCSFGTNmsHz5ckaMGMGjjz7KCSec0Og6rrrqKsrLyxkyZAg333yzd8QxatQoxowZwwknnMD3v//9mGG3i4uLmTp1qneiOaKwsJCLL76YcePGceKJJ3L55Zcf0tARN9xwAyNGjGD48OFMmDCBUaNGcfnll9OvXz/vf5wff/xxsrKyePjhh/nud7/LiBEj8Pl8/OhHP0q4zpKSEv74xz96/wqX6MS7Mab1iDOiavtRVFSk8dftr1u3jiFDhrT4cx0JYwi1J9FxJ+s9bWntbVybCIs7ddpKzCKyQlWLmqrX4Y4UjDHGNMySgjHGGI8lBWOMMZ4jJim0t3MjpmH2XhrTeo6IpJCVlUVpaak1JkcAVaW0tJSsrKzWDsWYDumI+PFanz592L59O3v37m3R9VZXV7fLxqm9x52VlUWfPn1aOxxjOqQjIin4/X4GDhzY4utdunRpu/hLyHgWtzHmcB0R3UfGGGNahiUFY4wxHksKxhhjPO1umAsR2Qs0Pqpcyzka2Jei52pJFnfqtMeYweJOpbYSc39V7dZUpXaXFFJJRJY3Z6yQtsbiTp32GDNY3KnU3mK27iNjjDEeSwrGGGM8lhQat6C1AzhMFnfqtMeYweJOpXYVs51TMMYY47EjBWOMMZ4OmxREpK+ILBGRtSKyRkSudctvFZEdIrLSnb4RtcyNIrJRRNaLyNdbMfbNIrLajW+5W9ZFRF4RkQ3u7VFuuYjIPW7cq0SksJViPj5qm64UkQMicl1b3N4i8pCI7BGRD6PKDnn7ishFbv0NInJRK8T8XyLykRvXIhHp7JYPEJGqqG1+f9QyY93P1kb3dUkrxH3InwkRmeqWbRSR2cmMuZG4/y8q5s0istItbzPbu1lUtUNOQE+g0J3PBz4GhgK3AtcnqD8U+ADIBAYCnwBprRT7ZuDouLI7gNnu/GzgN+78N4AXAAG+BrzdBrZ9GvAZ0L8tbm/gFKAQ+PBwty/QBdjk3h7lzh+V4pjPBNLd+d9ExTwgul7cet5xX4e4r+usVtjWh/SZcKdPgK8AGW6doamOO+7x3wI3t7Xt3Zypwx4pqOouVX3PnT8IrAN6N7LINGChqtao6qfARmBc8iNttmnAI+78I8C3o8ofVcdbQGcR6dkaAUaZAnyiqo39CLHVtreqvgHsTxDPoWzfrwOvqOp+Vf0ceAWYmsqYVfVlVQ26d98CGh161o27QFXfUqfFepS615kUDWzrhjT0mRgHbFTVTapaCyx06yZNY3G7e/vfA55obB2tsb2bo8MmhWgiMgAYA7ztFs10D7kfinQT4CSMbVGLbafxJJJMCrwsIitEpNgt66Gqu9z5z4Ae7nxbijvifGK/MG19e8Ohb9+2Fv+lOHuiEQNF5H0ReV1EJrplvXHijGjNmA/lM9HWtvVEYLeqbogqa+vb29Phk4KI5AFPA9ep6gHgPuBYYDSwC+cwsK05WVULgbOAq0XklOgH3b2ONnlZmYhkAGcDT7lF7WF7x2jL2zcREZkDBIESt2gX0E9VxwCzgMdFpKC14kug3X0m4lxA7E5PW9/eMTp0UhARP05CKFHVvwKo6m5VDalqGHiQui6LHUDfqMX7uGUpp6o73Ns9wCKcGHdHuoXc2z1u9TYTt+ss4D1V3Q3tY3u7DnX7ton4ReRi4JvADDeZ4Xa/lLrzK3D64we78UV3MbVKzIfxmWgT2xpARNKB6cD/Rcra+vaO12GTgtvv90dgnareFVUe3d/+HSBydcFzwPkikikiA4FBOCeJUkpEckUkPzKPczLxQze+yBUuFwHPuvPPAT90r5L5GlAW1Q3SGmL2otr69o5yqNv3JeBMETnK7f440y1LGRGZCvwHcLaqVkaVdxORNHf+KzjbdpMb9wER+Zr7/fghda8zlXEf6mfiXWCQiAx0j0TPd+u2htOBj1TV6xZq69u7ntY+091aE3AyThfAKmClO30D+DOw2i1/DugZtcwcnCy/nla6SgDnCosP3GkNMMct7wosBjYArwJd3HIB7nXjXg0UteI2zwVKgU5RZW1ue+MkrV1AAKef97LD2b44/fgb3emSVoh5I05fe+Tzfb9b9xz3s7MSeA/4VtR6inAa4U+AP+D+wDXFcR/yZ8L97n7sPjanNT4jbvmfgB/F1W0z27s5k/2i2RhjjKfDdh8ZY4ypz5KCMcYYjyUFY4wxHksKxhhjPJYUjDHGeCwpmDZHRLpGjSj5WdyImRnNXMfDInJ8E3WuFpEZLRN12yAiy0RkdGvHYdovuyTVtGkicitQrqp3xpULzuc33CqBtVEisgyYqaorWzsW0z7ZkYJpN0TkOHH+/6IE58dAPUVkgYgsF+c/MW6OqrtMREaLSLqIfCEi80XkAxH5t4h0d+vcLiLXRdWfLyLviDMu/wS3PFdEnnaf9y/uc9XbExeRr7qDna0QkRdEpIeI+N37J7t1/ktEfunO/1JE3hWRD0XkfjfJReK4y32etSJSJM5/IWxwE2RkO6wRkYUisk5EnhSR7AQxneW+3vfEGes/NyqOteIMOPebFn2TTLtnScG0NycA/62qQ9UZA2q2qhYBo4AzRGRogmU6Aa+r6ijg3zi/NE5EVHUccAMQSTDXAJ+p6lDgVzij6cYuJJIJ/A44R1XHAo8Bv1LVAHAJsEBEzgQmA7e7i/1OVb8KjHDjix5Wu8p9TX8EngF+5NYrFvePcnD+W+BuVR0CVANXxsXUHed/H6aoM3jiKuBaEemB8+vfYao6Evh1A9vCdFCWFEx784mqLo+6f4GIvIczfMAQnMYyXpWqRoaNXoHzpyeJ/DVBnZNxxudHVSNDi8QbAgwDXhXn37Zm4w7Qpqqr3OWfBS51EwXAFBF5B2e4klPd5SMi4/asBlarM0BcNc6fK0UGUPtUnf9vACcJnRwX0wScbfGmG9MM9zXtB8LAgyLyHaCigW1hOqj01g7AmEPkNWIiMgi4Fhinql+IyGNAVoJlaqPmQzT8ua9pRp1EBFilqhMbeHw4UAZEuq1ycMa5KVTVHSJye1zckTjCUfOR+5G44k8Gxt8X4EVV/UG9YEWKgDOA7wJX4QzWZwxgRwqmfSsADuKMNBn5t7OW9i+cf9FCREaQ+EhkLdBbRMa59TJEZJg7fx6QB0wC7hVnHP1snAZ+nzgj3p5zGHENFJGvuvPfB5bFPf4mcKo7Kmfk3Mgg9/kKVPXvwE9J0B1mOjY7UjDt2Xs4DfJHwBacBryl/R54VETWus+1Fmev36OqNSJyLnCP2+inAb8Vkb045yEmqepOEXkA53zIZSLyiLuuXdT949+hWAfMck96rwYWxMW0W0QuA/4v6jLem4Aq4K/ueRAfzp++GOOxS1KNaYQ4f5qSrqrVbnfVy8Agrfvv49aI6TjgL6pqv0cwLc6OFIxpXB6w2E0OAlzZmgnBmGSzIwVjjDEeO9FsjDHGY0nBGGOMx5KCMcYYjyUFY4wxHksKxhhjPJYUjDHGeP4/U6oD16E5VDAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "naive_bayes = MultinomialNB()  \n",
    "train(naive_bayes, X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00       484\n",
      "           1       0.99      0.97      0.98        95\n",
      "\n",
      "   micro avg       0.99      0.99      0.99       579\n",
      "   macro avg       0.99      0.98      0.99       579\n",
      "weighted avg       0.99      0.99      0.99       579\n",
      "\n",
      "Accuracy: 0.9930915371329879\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAF3JJREFUeJzt3X20XXV95/H3x4QHFQzVgNUkEBSoUlHBFHBYS5kRKTAaXNYHMqLFoaS10upobZ1VRyktWnW0YwccTYWiqCA41pXWWHxCMlrRxOGhgEIj8pCAw3MQwfDgd/7Y+5LjJdn35Hr3veeG92utuzh7n9/Z+7t/3JzP3ft3zm+nqpAkaWseN9MFSJJGm0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBoVklyVZLDJ2izZ5J7k8yZprJ6l+T6JEe0j09J8umZrkmPHQaFpkT7RnZ/+wb9/5KcnWSXqd5PVf1mVX1zgjY3VtUuVfXwVO+/fZN+sD3Ou5P8S5IXTvV+pFFiUGgqvbyqdgEOApYA7xrfII3Z/nv3ufY45wMXARfMcD1TLsncma5Bo2O2/4PVCKqqDcCXgecAJPlmktOSfBu4D3hGknlJzkxyS5INSf5q8FJRkpOS/CDJT5NcneSgdv3gJZiDk6xNck97FvPhdv3iJDX2Zpfk6UlWJrkzybokJw3s55Qk5yf5VLuvq5IsGfI4HwI+AyxIsvvANl+W5LKBM47nDjy3KMkXktyW5I4kp7frn5nkG+2625N8Jsluk+n/JMe2+78nyY+SHDW+7waO/dPj+uzEJDcC30jy5SQnj9v25Ule2T5+VpKvtv16TZLXTKZejT6DQlMuySLgGODSgdWvB5YDuwI3AGcDDwH7AAcCRwK/177+1cApwBuAJwFLgTu2sKuPAB+pqicBzwTO30pJ5wHrgacDrwLem+Q/DDy/tG2zG7ASOH3I49yxrfEO4K523YHAWcDvA08BPg6sTLJTG4T/1B7/YmBBu1+AAO9ra3w2sKjtg22S5GDgU8A72uN5EXD9Nmzixe3+fxs4F1g2sO39gb2ALyV5IvBV4LPAHsBxwEfbNtrOGBSaSl9McjfwLeBi4L0Dz51dVVe1f4U/mSZI3lpVP6uqW4G/oXmzgSYwPlBVa6qxrqpu2ML+HgT2STK/qu6tqkvGN2hD6zDgz6rq51V1GfAJmjf4Md+qqlXtmMY5wPMmOM7XtMd5P3AS8Kr2uKAJw49X1Xer6uGq+iSwCTgUOJgmCN7RHvfPq+pbAO0xfrWqNlXVbcCHad60t9WJwFnttn5RVRuq6ofb8PpT2truB/4BeH6SvdrnXgd8oao2AS8Drq+qv6+qh6rqUuB/A6+eRM0acQaFptIrqmq3qtqrqv6wfbMZc9PA472AHYBb2sszd9P85b1H+/wi4EdD7O9EYD/gh0nWJHnZFto8Hbizqn46sO4Gmr/mx/xk4PF9wM5J5iZ5XTtofW+SLw+0Ob+qdgOeClwJvGDcsb197LjaY1vU1rEIuGEgVB6R5KlJzmsvw90DfJpmDGRbDdt3W/PI/6e2z77E5gBfRnOpDZrjPGTccb4O+PVfYd8aUQ5YaboMTlN8E81f2fO39KbZPv/MCTdY9W/AsnZw/JXA55M8ZVyzm4EnJ9l1ICz2BDYMsf3PsPmNcUvP355kObA2yWer6pa29tOq6rTx7dtPR+2ZZO4Wjvu9NH10QFXdmeQVDHkJbJyuvvsZ8ISB5S29qY+fTvpc4D1JVgM70wzej+3n4qp66SRq1CzjGYWmXfuG+hXgQ0melORx7WDu2KWWTwB/kuQFzYekss/A5Y9HJDk+ye5V9Qvg7nb1L8bt6ybgX4D3Jdm5HVg+keYv9qk4lmuAC4E/bVf9HfAHSQ5pa39ikv+YZFfge8AtwF+363dOclj7ul2Be4GNSRbQjDFMxpnAG5O8pO3XBUme1T53GXBckh3aAftXDbG9VTRnD6fSfNprrH//Cdgvyevb7e2Q5LeSPHuSdWuEGRSaKW8AdgSuphkI/jzwNICqugA4jWag9KfAF2nGNcY7Crgqyb00A9vHjbvcNWYZzeDxzTTX3d9TVV+bwmP5ILA8yR5VtZZm3OL09rjWAScAtGMgL6cZwL+RZoD9te02/oLmY8UbaS73fGEyhVTV94A30oz5bKQZKxoL2f9Gc7ZxV7u/zw6xvU1tLUcMtm/Pzo6kuSx1M83lu/cDO02mbo22eOMiSVIXzygkSZ0MCklSJ4NCktTJoJAkdZp136OYP39+LV68eKbLkKRZ5fvf//7tVbX7xC0fbdYFxeLFi1m7du1MlyFJs0qSLU2DMxQvPUmSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTr0FRZKzktya5MqtPJ8kf5vmHsZXpL0nsiRptPR5RnE2zTTQW3M0sG/7sxz4Xz3WIkmapN6CoqpWA3d2NDkW+FR7T+RLgN2SPG2i7T7wwFRVKEkaxkyOUSzgl++jvJ5fvo/xI5IsT7I2ydpbbrlrWoqTJDVmxRQeVbUCWAHwjGcsqWuvneGCJGkEPPnJMH9+//uZyaDYACwaWF7IEDe8B1i9upd6JGnW2LSpCYply/rf10wGxUrg5CTnAYcAG6vqloleNHcuHHhg77VJ0ki74Qa4s2sUeAr1FhRJzgUOB+YnWQ+8B9gBoKo+BqwCjqG5+fx9NDeElySNmN6Coqo6T4iqqoA397V/SdLU8JvZkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI69RoUSY5Kck2SdUneuYXn90xyUZJLk1yR5Jg+65EkbbvegiLJHOAM4Ghgf2BZkv3HNXsXcH5VHQgcB3y0r3okSZPT5xnFwcC6qrquqh4AzgOOHdemgCe1j+cBN/dYjyRpEub2uO0FwE0Dy+uBQ8a1OQX4SpI/Ap4IHLGlDSVZDiwH2GOPPae8UEnS1s30YPYy4OyqWggcA5yT5FE1VdWKqlpSVUvmzdt92ouUpMeyPoNiA7BoYHlhu27QicD5AFX1HWBnYH6PNUmStlGfQbEG2DfJ3kl2pBmsXjmuzY3ASwCSPJsmKG7rsSZJ0jbqLSiq6iHgZOBC4Ac0n266KsmpSZa2zd4OnJTkcuBc4ISqqr5qkiRtuz4Hs6mqVcCqcevePfD4auCwPmuQJP1qZnowW5I04gwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktRp7rANkywA9hp8TVWt7qMoSdLoGCookrwfeC1wNfBwu7qAzqBIchTwEWAO8Imq+usttHkNcEq7vcur6j8NW7wkqX/DnlG8AviNqto07IaTzAHOAF4KrAfWJFlZVVcPtNkX+K/AYVV1V5I9hi9dkjQdhh2juA7YYRu3fTCwrqquq6oHgPOAY8e1OQk4o6ruAqiqW7dxH5Kkng17RnEfcFmSrwOPnFVU1R93vGYBcNPA8nrgkHFt9gNI8m2ay1OnVNU/D1mTJGkaDBsUK9ufPva/L3A4sBBYneSAqrp7sFGS5cBygD322LOHMiRJWzNUUFTVJ5PsSHsGAFxTVQ9O8LINwKKB5YXtukHrge+22/pxkmtpgmPNuP2vAFYA7LffkhqmZknS1BhqjCLJ4cC/0QxOfxS4NsmLJnjZGmDfJHu3IXMcjz4r+SLN2QRJ5tME0XXDFi9J6t+wl54+BBxZVdcAJNkPOBd4wdZeUFUPJTkZuJBm/OGsqroqyanA2qpa2T53ZJKxj92+o6rumPzhSJKm2rBBscNYSABU1bVJJvwUVFWtAlaNW/fugccFvK39kSSNoGGDYm2STwCfbpdfB6ztpyRJ0igZNijeBLwZGPs47P+hGauQJG3nhv3U0ybgw+2PJOkxpDMokpxfVa9J8q80czH9kqp6bm+VSZJGwkRnFG9p//uyvguRJI2mzu9RVNUt7cPbgZuq6gZgJ+B5wM091yZJGgHDTgq4Gti5vSfFV4DXA2f3VZQkaXQMGxSpqvuAVwIfrapXA7/ZX1mSpFExdFAkeSHN9ye+1K6b009JkqRRMmxQvJXmBkP/0E7D8Qzgov7KkiSNimG/R3ExcPHA8nVs/vKdJGk7NtH3KP5HVb01yT+y5e9RLO2tMknSSJjojOKc9r//ve9CJEmjqTMoqur77cO1wP1V9QuAJHNovk8hSdrODTuY/XXgCQPLjwe+NvXlSJJGzbBBsXNV3Tu20D5+Qkd7SdJ2Ytig+FmSg8YWkrwAuL+fkiRJo2TY+1G8Fbggyc1AgF8HXttbVZKkkTHs9yjWJHkW8Bvtqmuq6sH+ypIkjYqhLj0leQLwZ8BbqupKYHESpx6XpMeAYcco/h54AHhhu7wB+KteKpIkjZRhg+KZVfUB4EGAdibZ9FaVJGlkDBsUDyR5PO00HkmeCWzqrSpJ0sgY9lNP7wH+GViU5DPAYcAJfRUlSRodEwZFkgA/pLlp0aE0l5zeUlW391ybJGkETBgUVVVJVlXVAWy+aZEk6TFi2DGK/5vkt3qtRJI0koYdozgEOD7J9cDPaC4/VVU9t6/CJEmjYdig+O1eq5AkjayJ7nC3M/AHwD7AvwJnVtVD01GYJGk0TDRG8UlgCU1IHA18qPeKJEkjZaJLT/u3n3YiyZnA9/ovSZI0SiY6o3hkhlgvOUnSY9NEQfG8JPe0Pz8Fnjv2OMk9E208yVFJrkmyLsk7O9r9TpJKsmRbD0CS1K/OS09VNWeyG04yBzgDeCmwHliTZGVVXT2u3a7AW4DvTnZfkqT+DPuFu8k4GFhXVddV1QPAecCxW2j3l8D7gZ/3WIskaZL6DIoFwE0Dy+vbdY9o78O9qKo6pwZJsjzJ2iRrN268beorlSRtVZ9B0SnJ44APA2+fqG1VraiqJVW1ZN683fsvTpL0iD6DYgOwaGB5YbtuzK7Ac4BvtlODHAqsdEBbkkZLn0GxBtg3yd5JdgSOA1aOPVlVG6tqflUtrqrFwCXA0qpa22NNkqRt1FtQtN+7OBm4EPgBcH5VXZXk1CRL+9qvJGlqDTsp4KRU1Spg1bh1795K28P7rEWSNDkzNpgtSZodDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR16jUokhyV5Jok65K8cwvPvy3J1UmuSPL1JHv1WY8kadv1FhRJ5gBnAEcD+wPLkuw/rtmlwJKqei7weeADfdUjSZqcPs8oDgbWVdV1VfUAcB5w7GCDqrqoqu5rFy8BFvZYjyRpEvoMigXATQPL69t1W3Mi8OUtPZFkeZK1SdZu3HjbFJYoSZrISAxmJzkeWAJ8cEvPV9WKqlpSVUvmzdt9eouTpMe4uT1uewOwaGB5YbvulyQ5Avhz4MVVtanHeiRJk9DnGcUaYN8keyfZETgOWDnYIMmBwMeBpVV1a4+1SJImqbegqKqHgJOBC4EfAOdX1VVJTk2ytG32QWAX4IIklyVZuZXNSZJmSJ+XnqiqVcCqcevePfD4iD73L0n61Y3EYLYkaXT1ekYhSerPpk1w7bX978egkKRZaN48+MlPYPXqYV+x6xMnuy+DQpJmod12g0MO2ZZXzJkz2X05RiFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSerUa1AkOSrJNUnWJXnnFp7fKcnn2ue/m2Rxn/VIkrZdb0GRZA5wBnA0sD+wLMn+45qdCNxVVfsAfwO8v696JEmT0+cZxcHAuqq6rqoeAM4Djh3X5ljgk+3jzwMvSZIea5IkbaO5PW57AXDTwPJ64JCttamqh5JsBJ4C3D7YKMlyYHm79OCSJb92fS8Vzzqb5sFOG2e6itFgX2xmX2xmX2x2z8LJvrLPoJgyVbUCWAGQZG3VXUtmuKSR0PTFffYF9sUg+2Iz+2KzJGsn+9o+Lz1tABYNLC9s122xTZK5wDzgjh5rkiRtoz6DYg2wb5K9k+wIHAesHNdmJfC77eNXAd+oquqxJknSNurt0lM75nAycCEwBzirqq5KciqwtqpWAmcC5yRZB9xJEyYTWdFXzbOQfbGZfbGZfbGZfbHZpPsi/gEvSeriN7MlSZ0MCklSp5ENCqf/2GyIvnhbkquTXJHk60n2mok6p8NEfTHQ7neSVJLt9qORw/RFkte0vxtXJfnsdNc4XYb4N7JnkouSXNr+OzlmJursW5Kzktya5MqtPJ8kf9v20xVJDhpqw1U1cj80g98/Ap4B7AhcDuw/rs0fAh9rHx8HfG6m657Bvvj3wBPax296LPdF225XYDVwCbBkpuuewd+LfYFLgV9rl/eY6bpnsC9WAG9qH+8PXD/TdffUFy8CDgKu3MrzxwBfBgIcCnx3mO2O6hmF039sNmFfVNVFVXVfu3gJzXdWtkfD/F4A/CXNvGE/n87iptkwfXEScEZV3QVQVbdOc43TZZi+KOBJ7eN5wM3TWN+0qarVNJ8g3ZpjgU9V4xJgtyRPm2i7oxoUW5r+Y8HW2lTVQ8DY9B/bm2H6YtCJNH8xbI8m7Iv2VHpRVX1pOgubAcP8XuwH7Jfk20kuSXLUtFU3vYbpi1OA45OsB1YBfzQ9pY2cbX0/AWbJFB4aTpLjgSXAi2e6lpmQ5HHAh4ETZriUUTGX5vLT4TRnmauTHFBVd89oVTNjGXB2VX0oyQtpvr/1nKr6xUwXNhuM6hmF039sNkxfkOQI4M+BpVW1aZpqm24T9cWuwHOAbya5nuYa7MrtdEB7mN+L9cDKqnqwqn4MXEsTHNubYfriROB8gKr6DrAzMH9aqhstQ72fjDeqQeH0H5tN2BdJDgQ+ThMS2+t1aJigL6pqY1XNr6rFVbWYZrxmaVVNejK0ETbMv5Ev0pxNkGQ+zaWo66azyGkyTF/cCLwEIMmzaYLitmmtcjSsBN7QfvrpUGBjVd0y0YtG8tJT9Tf9x6wzZF98ENgFuKAdz7+xqpbOWNE9GbIvHhOG7IsLgSOTXA08DLyjqra7s+4h++LtwN8l+S80A9snbI9/WCY5l+aPg/nteMx7gB0AqupjNOMzxwDrgPuANw613e2wryRJU2hULz1JkkaEQSFJ6mRQSJI6GRSSpE4GhSSpk0EhjZPk4SSXJbkyyT8m2W2Kt39CktPbx6ck+ZOp3L401QwK6dHur6rnV9VzaL6j8+aZLkiaSQaF1O07DEyaluQdSda0c/n/xcD6N7TrLk9yTrvu5e29Ui5N8rUkT52B+qVf2Uh+M1saBUnm0Ez7cGa7fCTNXEkH08znvzLJi2jmGHsX8O+q6vYkT2438S3g0KqqJL8H/CnNN4SlWcWgkB7t8UkuozmT+AHw1Xb9ke3Ppe3yLjTB8Tzggqq6HaCqxu4HsBD4XDvf/47Aj6enfGlqeelJerT7q+r5wF40Zw5jYxQB3teOXzy/qvapqjM7tvM/gdOr6gDg92kmopNmHYNC2or2roF/DLy9ncr+QuA/J9kFIMmCJHsA3wBeneQp7fqxS0/z2DyF8+8izVJeepI6VNWlSa4AllXVOe0U1d9pZ+m9Fzi+nan0NODiJA/TXJo6geauahckuYsmTPaeiWOQflXOHitJ6uSlJ0lSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHX6/58O/YYvDyBZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluate(naive_bayes, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# k-NN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameter tuning with GridsearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 60 candidates, totalling 180 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers.\n",
      "[Parallel(n_jobs=10)]: Done  30 tasks      | elapsed:   57.7s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best options\n",
      "=======================================\n",
      "n_neighbors: 25\n",
      "weights: 'distance'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Done 180 out of 180 | elapsed:  5.4min finished\n"
     ]
    }
   ],
   "source": [
    "param_grid={'weights': ['uniform', 'distance'], 'n_neighbors': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, \n",
    "                                            14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]}\n",
    "best_parameters = hyper_parameter_tuning(param_grid , KNeighborsClassifier(), 3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training with best parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XmcFPWd//HXp7vnZIZBUImgMGyCCnIzAS8iLGowj0TjFUXUoEaMK8mq0RUlP80aSYxJjCZxvVbdYFAi8YBdbwnEGCMKCijgAQqKIAhEYO4+vr8/urronum5oLtnhnk/59GPqfp2HZ/6dnV9ur5V/W1zziEiIgIQaO8ARESk41BSEBERn5KCiIj4lBRERMSnpCAiIj4lBRER8SkpiIiIT0lBRER8WUsKZvagmW01s3eaeN7M7LdmttbMVprZqGzFIiIirRPK4rL/B/g9MLuJ508BBnqPscDd3v9mHXjgga68vLxtkezYAZ9+CvX1kJ8PfftCz54tzlZVVUW3bt3atq4sxNFWLcadozha1CCO2l69KOzTp11jaGtdZGwfyfFr0mTcHXTfSMSR8ffkXsbRFhmJOQNxLFu2bJtz7qAWJ3TOZe0BlAPvNPHcvcDkpPH3gENaWubo0aNdm/zxj84VFzsHex7FxfHyFixatKht68pSHG3VbNw5jKNZaeKIFBTkNo4M1EVG9pF2eE3Sxt2B941EHBl9T+5DHG2xzzFnKA5gqWvFcdtcFvs+MrNy4P+cc0PSPPd/wK3OuVe88YXAdc65pc0ts6Kiwi1d2uwkqcrLYcOGxuUFBXD00c3O+sUXX9CjR4/Wr6s5r70GdXV7FUdbNRt3DuNoVkeIIwMxZGQfaYe6SBt3R3hNWojjiyOPzNx7ch/iaEt97PM+0lQc/fvD+vWtXoyZLXPOVbQ0XTabjzLGzKYB0wB69+7N4sWLWz3vCR9/jKUpd3V17PznP5udNxqL8UUL07RWWV3dXsfRVs3Fncs4mtMR4shEDJnYR9qjLtLF3RFek5biiEajGXtP7kscbamPfd1Hmozj44/5axuOha3WmtOJvX3QEZqP+vdPPe1KPPr3b3HWjJ6q7kMcbdVs3DmMo1kdIY4MxJCRfaQd6iJt3B3hNWkhjpw2H2WoPvY55gzFQSubj9rzltQFwIXeXUhHAzudc5szvpZZs6C4OLWsuDhenkuKo8U4ogUFuY2jA9eF4lAc7RZHazLH3jyAR4HNQBjYCFwCfB/4vve8AXcB64C3gYrWLLfNZwrOxS/I9O/vnFn8fysv0GT8U8lextFWLcadozha1CCOVTNntnsMOb+I2CCOmJmL9evnon982EWiEReJRlw4GnbhaNjVR+pdfaTe1UXqXF2kztWGa11tuNbVhGtcTbjGVddXu+r6aldVX+Wq6qtcZV2lq6yrdLvrdrvddbvdrtpdblftLvdFzRfupb+85HZU73A7qne4bVXb3LaqbW5r5VZ3/28vcv1+FHR2E67fj4Lugd9e7HbV7nKVdZWuur7a1YRrXF2kzoWjYReNRTOz7c3UR8PXJadnCs3E0RYZuxlhH+OglWcKWbum4Jyb3MLzDrgiW+tPMWVK/NHeFEezcWxdvJjB7RxDpjjniLkYMRfDsWc45mJEY1EisQhRF/WHH//y5/zyh2E274ZDSiJcPWAL3/rnOm9h3kITDcv7OG4YkViE7TXb/XgNY8F7C7hh16PUlkYB+Lg0yhW7HuGfywbzrSO+5W+XNWjhDgQCBC1IKBDyHwELEAqECAaCBCxAwAIY5g8HLIBZupbyuDnDYOaV8PFO6FcGs4ZBe+yxHSWOXL5nO8WFZuncEgfClAOji/Lo249yy8u38OnuT+lb2pfz+5zPUVVHpRw4kg8ehmFmjf639Fxz5rw9h5kLZ/Lxzo/pV9aPWRNnMWXonjdf8sG94QE+Gosf1LdUbkk5wEddfFsT8zc8iJqZv13+wXjhDdREagDYVLmJ/7fo/1EQKuCMQWcQczHC0TCRWIRwzPvvjdfH6olEk8pj4ZTxSCxCfbS+0XOfbPqEA4MHEonuWcYDbz1AbaQ2JdbaSC03v3wzX9R+QX4wn7xgHgXBAvKD+fFHKJ+8QB75gXxCwRB5gTxCgRB5wTzyAvFHYtpQIEQgsKfFOmABAgQIBvYklKAFeeLdJ7jyuSv9+tiwcwOXLriU+kg95a6cmnDNntfHz3iZ96d3/sQVz1yREse0BdOIRCOcO+TcRvtb4rVt7b7XFi3tp5mU1VtSs6HNt6Tug8WLFzN+/PicrCuTWop7X3ewhp96E8MND0KJg2RinvpYPTXhGmojtTzzwTPcueRO6qJ7brXLszymj53OhPIJ/psr8QnTzAgFQvGyQIAgQSxgBPCSRyB+gGn03wKEgiFCFvIPxgGLH5jmvzuf/3jpP/w3PUBRqIifTfwZJ/3LSdRE4rHWReuoj9RTF6kjHA1TH/OGY2E+Xvcxvfr1oj5aTzgaJhwLUx+t9x+1kVrqo/X+Muqj9dRGvbJIHfXRelZ9vsqvp4YCFvATzP7ATybpHoF40lm5ZWXKfpFQFCri+J7H071X99Qzj6TX2swIWtAfTrzeQQvuGW+wfyTPlzzPr//xa3bW7WwUR6+iXvzm67+JJ7xEUvSSXmJbEmUFoQI+Wv4RA0cN9Pfj5A88QKMPQcllZsZjqx5j+jPTU/bT4rxi7vvWfW1637b2llQlhWZkOinkKts3F/ect+cw7X+nUR2u9suKQkX88qRfctKXT6I6XE1VfRWV4Uoq6yqprI8/qsJVVNVXURWuojZc6x8wayO11EZrqQnXUBeti4+H42W1kXh5Ytr21vDNVxepy+onzeRP1AWh+HBhsJD8kFcWLKAgWMBf1v+lyWX8cOwP93z6DuSlfhoP5JEXbPq5xEEq8ck9uXzdynUMGT3EX0bQgoz977F8uvvTRjH0Le3L4qmL/STmJ7mk5JdIeuFYeM9zkfTTNpovMR6Jj/9j4z+arI/eBb0J5geJxWLE2PPhpMXxdkysIQtRECogL5jnJ4zk4cQZWNrhQB5Pf/A0VeGqRsvtX9af9Veub3Uc+9X3FPYHDQ/GG3ZuYNr/xk9Fzxh0hn8ATRxM/XHvoFsb3lNeF63b81yDR120js1bNnPbp7fFxyN1/jLqInWs/2I9URdNia0mUsP0Z6e3aXsKQ4UUhYooyitKGe6W141eRb0oyiuKl4W85/NSh6998doml/3AqQ/4TUwxF8M5Fz8jIekN7+Jv+mgs6jfxpEzvUpurkputEtPfvfTuJmOYOW5mysE8cQBPlCWGP3n3EwYPH5xSlvjf2iaEMfePafJgfN1x17VqGW21LW8b3Qu6p5TNOH4G//Fi4zOnGcfP8F9LSG2ySXyo3JeyZA7HCf9zApt2b2r0XJ/SPjw4/EG+MvIrflly82FTZWbmryvRBBh1UXCpZ70Nmwa/+eg3+azys0ZxHFR8EHd9466Us8Jw1BuOJQ17j00bN1HWuywlaYajYcJRbzgWH66J1LCrbteeZXrLT5cQAD7e+XHa8n2lpJAD4WiY61+6PuXTOUB1uJqp86cydf7UfVp+XiDPPygVhAqwiFESK4kfnEL5FOUV0aOwB/mhfNYlLl6m8bN//Zl/8C4IFfgH8cJQIcV5xf4BvSBY4J/mNqXhAbFhu/odr92R9kDYp7QPE8ontOrg0fC5lE/9ybPYnuX4cRjMf29++oNPSR8uHH5h2nkaLj9QFODgbgenbGeiGS25DhpuP+ypox8d+yNmLpzZ6GD8o2N/lNJ+nm67/W128WEza/ZCc2J7YrEYlfWVKc+d/OWTqYvUcfs/bmdz5WYOKT2Eq4++mpP/5WSq6qv86wGBpDvZG17TAfx9I11Z8nBTZTedcFPKNYVEfdx0wk2EdoboUdjD/+Tf1H/nHP5fUkJI1EeQIDGLxZuMYvGmI7/+vGmuOeYafrzoxylnuYWhQq477jqG9x7eqI4T8yfqODH+UfAjBowYkHKhv2HdJWtYPu6hcWnfK/3K+jUqywQlhQxzzvntxVXhKtZ/sZ7HVj3GJ7s+aXKea465ptGn0uS21sQBvzBUuGc8UEBBXgH5gfz4m9Xht5OufWstR4w+IqU9NXEXyIh7R7Bx18ZGMRzW/TCmj4mfLTQ81W5pPPEG9MeT3ohNzTPjuBmN2vMLAgXccPwNFIYK0x4w2nKQaeqAnPyGu2XCLSkXEgGKQ8Xc8q+3cFj3w/xtSd7O5G0E2BjcyCElhzTa7uTtbWn4vCHnkR/I5+ev/JxNuzfRp7QP1x9/PWcNPmvPhcuk9uh025zu03FTBx8z49PQpwzoMaDRc1cdcxVXHXNVo3lzadroaXTL75a2qXXx4sX0Ku6VlfU2fH2vOuYqDu52MD9e9GM+2fkJh5Udxk8n/JTJQyanJOemEnVifGNwI32692m0b6T7n9iHkpNauvdKcV4xsyZm53sKSgr7KBqL+hcUK+srqYnUEIvFeGPTG8xdNZcX1r1AJBahIFiQ9uJZ39K+XHPsNY3au5PvTkkebu5/wsfBj+nfo3/aeG898dZG1xSK84r5+Yk/p7SgNPMV1ITpY6dzQPEBKW/88w85nyvG5OYuZYCLRl5Efih/n67zBCyQkXq7YswVOd12w8gL5uVsfW01ZeiUrN1d05R0HyQuGH4BFwy/YJ+WG7AAJfklez1/uvdKNu8+UlJoo0R7YU24ht31u6mP1gPxF74mUsNT7z7FH1f+kQ92fECPgh5cNOIizh92Pks3LWXmX2amnIoW5xXzi5N+Qe+S3jmLP7Ej5WoHaymW5PW2pU+rbMUg0hHlcj9VUmhB4o6LxB050Vj8Im3ijo7SUCkrt6xk9orZPPXuU9REahj5pZHc/vXbOfXwUwkGgtRGapk8ZDIHFh/IjYtu7HAHYxGRBCWFJMlNQVX1VdRF69jwRbzb7VAwlNLWXROu4fHVjzN7xWyWb1lOUaiI0488nQuHX8jQ3kOJxqJUh6sJWpD+Zf0pyiviwuEX+hcwRUQ6oi6dFJKbgirrK/2mILN4e2vAApQUpLYFrt2xlodXPsy8VfPYWbeTgT0H8tMJP+XMQWdSVliGc47q+mocjkNKDqG0oLRdLtaJiOyNLpUUIrEINeEaquqrqA5X+98iDQaC5AXzKAmlvxgUjoZ5ft3zzF4xm79/8nfyAnmcMvAULhx2IUcferR/0K+N1BKOhjmg6AB6FvUkFOhS1Ssi+4EuddTaUbOD7dXb47d+hvIptMJmp99at5Vn/v4Mj77zKFuqtvhfJpo8ZDIHddvzU6eJZFOUV0Tf0r4UhAqyvSkiIlnRpZKCc46CUPx+/6bEXIyXN7zM7BWzeXHdizgcEwZM4BfDf8G/lv8rwUAwZXlV9VUEA0H6lPahJL9ETUUi0ql1qaTQnB01O/jTO3/i4ZUPs2HnBnoV9eLsQ8/myq9fmfabgzXhGiKxCL2KenFA0QEpyUJEpLPq0knBOcfSzUuZvWI2T7//NHXROo7uezTXHXcdk74yibVvrW2UEMLRMDXhGkoLSjmo20HkB/PbKXoRkczrEkkhuXfSRH8u4ViY2Stms2bbGkrzSzlv6HlcMOwCjjjwiLTLiLkY1eFqQhaiX49+FOcVp51ORKQz2++TQsPeSTft3sQ1L14DwJCDh3Dbibfx7SO/Tbf8bk0uozpcTSwW46BuB1FWWNZiZ3AiIp3Vfp8UZi6c2ah3Uoh3f/vclOeavTDsnGNX3S7KCso4sPjADt1XjIhIJuz3SaGpPse3VW9rMiEkvo0M+N9GFhHpCvb7dpCm+hzvU9qnUVni28i1kVq+VPIl8oP5Sggi0qXs90lh1sRZjS4KJ35NKlmi6+vuhd0ZcMAAygrLchmmiEiHsN83HzXsKvqQ0kO4/vjrOWPQGYC+jSwikmy/Twqwp6voLZVbqApXURgq1LeRRUTS6BJJoSF9G1lEJL0ulxRqwjX0KOyhbyOLiKTRpZJCt7xulPQoafaLaiIiXVmXSgoNfzBHRERS7fe3pIqISOspKYiIiE9JQUREfEoKIiLiU1IQERGfkoKIiPiUFERExKekICIiPiUFERHxKSmIiIhPSUFERHxZTQpmNsnM3jOztWY2I83z/cxskZm9ZWYrzewb2YxHRESal7WkYGZB4C7gFGAwMNnMBjeY7MfAY865kcC5wH9lKx4REWlZNs8UxgBrnXMfOufqgbnAaQ2mcUB3b7gM2JTFeEREpAXZ7Dq7L/BJ0vhGYGyDaX4CvGBmPwC6ASdmMR4REWmBOeeys2Czs4BJzrnveeMXAGOdc9OTprnai+HXZnYM8AAwxDkXa7CsacA0gN69e4+eO3duVmJuqLKykpKSzvcbDIo7dzpjzKC4c6mjxDxhwoRlzrmKFid0zmXlARwDPJ80fj1wfYNpVgGHJY1/CBzc3HJHjx7tcmXRokU5W1cmKe7c6YwxO6e4c6mjxAwsda04dmfzmsIbwEAzG2Bm+cQvJC9oMM3HwEQAMxsEFAKfZzEmERFpRtaSgnMuAkwHngfWEL/LaJWZ3Wxmp3qT/Qi41MxWAI8CU72MJiIi7SCrv9HsnHsGeKZB2Y1Jw6uB47IZg4iItJ6+0SwiIj4lBRER8SkpiIiIT0lBRER8SgoiIuJTUhAREZ+SgoiI+JQURETEp6QgIiI+JQUREfEpKYiIiE9JQUREfEoKIiLiU1IQERGfkoKIiPiUFERExKekICIiPiUFERHxKSmIiIhPSUFERHxKCiIi4lNSEBERn5KCiIj4lBRERMSnpCAiIj4lBRER8SkpiIiIT0lBRER8SgoiIuJTUhAREZ+SgoiI+JQURETEp6QgIiI+JQUREfEpKYiIiE9JQUREfEoKIiLiU1IQERGfkoKIiPiymhTMbJKZvWdma81sRhPTfMfMVpvZKjN7JJvxiIhI80LZWrCZBYG7gJOAjcAbZrbAObc6aZqBwPXAcc65f5rZwdmKR0REWtbqMwUzO97MLvKGDzKzAS3MMgZY65z70DlXD8wFTmswzaXAXc65fwI457a2PnQREck0c861PJHZTUAFcIRz7nAz6wPMc84d18w8ZwGTnHPf88YvAMY656YnTfMU8D5wHBAEfuKcey7NsqYB0wB69+49eu7cuW3YxL1XWVlJSUlJTtaVSYo7dzpjzKC4c6mjxDxhwoRlzrmKlqZrbfPR6cBI4E0A59wmMyvdh/iS1z8QGA8cCrxsZkOdc18kT+Scuw+4D6CiosKNHz8+A6tu2eLFi8nVujJJcedOZ4wZFHcudbaYW9t8VO/ipxQOwMy6tWKeT4HDksYP9cqSbQQWOOfCzrmPiJ81DGxlTCIikmGtTQqPmdm9QA8zuxR4Cbi/hXneAAaa2QAzywfOBRY0mOYp4mcJmNmBwOHAh62MSUREMqxVzUfOuV+Z2UnALuAI4Ebn3IstzBMxs+nA88SvFzzonFtlZjcDS51zC7znTjaz1UAUuNY5t30ftkdERPZBi0nBu7X0JefcBKDZRNCQc+4Z4JkGZTcmDTvgau8hIiLtrMXmI+dcFIiZWVkO4hERkXbU2ruPKoG3zexFoCpR6Jz7YVaiEhGRdtHapPCE9xARkf1Yay80/8G7g+hwr+g951w4e2GJiEh7aFVSMLPxwB+A9YABh5nZd51zL2cvNBERybXWNh/9GjjZOfcegJkdDjwKjM5WYCIiknut/fJaXiIhADjn3gfyshOSiIi0l9aeKSw1s/8G/uiNTwGWZickERFpL61NCpcDVwCJW1D/BvxXViISEZF209qkEALudM7dDv63nAuyFpWIiLSL1l5TWAgUJY0XEe8UT0RE9iOtTQqFzrnKxIg3XJydkEREpL20NilUmdmoxIiZVQA12QlJRETaS2uvKVwJzDOzTd74IcA52QlJRETaS7NnCmb2VTP7knPuDeBI4E9AGHgO+CgH8YmISA611Hx0L1DvDR8D3ADcBfwT7zeTRURk/9FS81HQObfDGz4HuM859zjwuJktz25oIiKSay2dKQTNLJE4JgJ/SXqutdcjRESkk2jpwP4o8Fcz20b8bqO/AZjZV4CdWY5NRERyrNmk4JybZWYLid9t9IL3m8oQP8P4QbaDExGR3GqxCcg591qasvezE46IiLSn1n55TUREugAlBRER8SkpiIiIT0lBRER8SgoiIuJTUhAREZ+SgoiI+JQURETEp6QgIiI+JQUREfEpKYiIiE9JQUREfEoKIiLiU1IQERGfkoKIiPiUFERExKekICIivqwmBTObZGbvmdlaM5vRzHRnmpkzs4psxiMiIs3LWlIwsyBwF3AKMBiYbGaD00xXCvw7sCRbsYiISOtk80xhDLDWOfehc64emAuclma6nwK/AGqzGIuIiLRCNpNCX+CTpPGNXpnPzEYBhznnns5iHCIi0kqh9lqxmQWA24GprZh2GjANoHfv3ixevDirsSVUVlbmbF2ZpLhzpzPGDIo7lzpdzM65rDyAY4Dnk8avB65PGi8DtgHrvUctsAmoaG65o0ePdrmyaNGinK0rkxR37nTGmJ1T3LnUUWIGlrpWHLuz2Xz0BjDQzAaYWT5wLrAgKRntdM4d6Jwrd86VA68BpzrnlmYxJhERaUbWkoJzLgJMB54H1gCPOedWmdnNZnZqttYrIiJ7L6vXFJxzzwDPNCi7sYlpx2czFhERaZm+0SwiIj4lBRER8SkpiIiIT0lBRER8SgoiIuJTUhAREZ+SgoiI+JQURETEp6QgIiI+JQUREfEpKYiIiE9JQUREfEoKIiLiU1IQERGfkoKIiPiUFERExKekICIiPiUFERHxKSmIiIhPSUFERHxKCiIi4lNSEBERn5KCiIj4lBRERMSnpCAiIj4lBRER8SkpiIiIT0lBRER8SgoiIuILtXcAmRAOh9m4cSO1tbUZXW5ZWRlr1qzJ6DJzobPHXVhYyKGHHkpeXl57hyTS5ewXSWHjxo2UlpZSXl6OmWVsubt376a0tDRjy8uVzhx3SUkJ27dvZ+PGjQwYMKC9QxLpcvaL5qPa2lp69eqV0YQg7cPM6NWrV8bP+kSkdfaLpAAoIexH9FqKtJ/9Jim0p+3btzNixAhGjBjBl770Jfr27euP19fXt2oZF110Ee+9916z09x1113MmTMnEyGLiKS1X1xTaLM5c2DmTPj4Y+jXD2bNgilT9npxvXr1Yvny5QD85Cc/oaSkhGuuuSZlGucczjkCgfR5+KGHHmpxPVdcccVex5hNLW2biHQeXe9dPGcOTJsGGzaAc/H/06bFyzNs7dq1DB48mClTpnDUUUexefNmpk2bRkVFBUcddRQ333yzP+3xxx/P8uXLiUQi9OjRgxkzZjB8+HCOOeYYtm7dCsCPf/xj7rjjDn/6GTNmMGbMGI444gheffVVAKqqqjj//PMZPHgwZ511FhUVFX7CSnbttdcyePBghg0bxnXXXQfAZ599xmmnncawYcMYPnw4S5YsAeC2225jyJAhDBkyhN/97ndNbtuzzz7LMcccw6hRozjnnHOoqqrKeJ2KSHbtf2cKV14JaQ6Cvtdeg7q61LLqarjkErj//pTiomgUgkEYMQK8g3Fbvfvuu8yePZuKigoAbr31Vnr27EkkEmHChAmcddZZDB48OGWenTt3csIJJ3Drrbdy9dVX8+CDDzJjxoxGy3bO8frrr7NgwQJuvvlmnnvuOX73u99x8MEHM3/+fFasWMGoUaMazbdlyxaeeeYZVq1ahZnxxRdfAPEzkZNOOonp06cTiUSorq5myZIlzJkzhzfeeINIJMKYMWMYP348RUVFKdu2detWbr31VhYuXEhxcTGzZs3izjvv5IYbbtirehOR9tH1zhQaJoSWyvfRl7/8ZT8hADz66KOMGjWKUaNGsWbNGlavXt1onqKiIk455RQARo8ezfr169Mu+4wzzmg0zSuvvMJZZ50FwPDhwznqqKMazdezZ08CgQCXXnopTz75JN26dQNg8eLFXHbZZQCEQiG6d+/OK6+8wplnnklRURGlpaV8+9vf5m9/+1ujbXv11VdZvXo1xx57LCNGjGDOnDlNxi0iHdf+d6bQ0if68vJ4k1FD/fvD4sUpRTUZuN8/ccAF+OCDD7jzzjt5/fXX6dGjB+eff37aWy/z8/P94WAwSCQSSbvsgoKCFqdJJy8vj6VLl/Liiy8yb9487r77bl544QWgbXf+JG+bc45Jkybx8MMPt3p+Eel4snqmYGaTzOw9M1trZo3aP8zsajNbbWYrzWyhmfXPZjxA/KJycXFqWXFxvDzLdu3aRWlpKd27d2fz5s08//zzGV/HcccdxxNPPAHA22+/nfZMZPfu3ezatYtvfvOb/OY3v+Gtt94CYMKECdxzzz0ARKNRdu3axbhx43jyySepqamhsrKS+fPnM27cuEbLPPbYY/nrX//Khx9+CMSvbXzwwQcZ3z4Rya6snSmYWRC4CzgJ2Ai8YWYLnHPJR6m3gArnXLWZXQ7cBpyTrZiAPXcZZfDuo9YaNWoUgwcP5sgjj6R///4cd9xxGV/HD37wA8477zwGDx7sP8rKylKm2blzJ2eccQZ1dXXEYjFuv/12AH7/+99z6aWXcu+99xIKhbj33nsZM2YMkydP5qtf/SoAl19+OUOHDmXt2rUpy+zduzcPPPAA55xzjn8b7s9+9jMGDhyY8W0UkSxK3E6Y6QdwDPB80vj1wPXNTD8S+HtLyx09erRraPXq1Y3KMmHXrl1ZWW42hcNht3XrVuecc++//74rLy934XC4naNqneT6ztZrmmmLFi1q7xD2iuLOnY4SM7DUteLYnc1rCn2BT5LGNwJjm5n+EuDZLMbTJVRWVnLyyScTi8Vwzvmf+kVEWqNDHC3M7HygAjihieenAdMg3kyxuMEF4bKyMnbv3p3xuKLRaFaWm03BYJBFixYRDAb9ss6yDcn1XVtb2+h17ogqKys7RZwNKe7c6WwxZzMpfAocljR+qFeWwsxOBGYCJzjn0t4X6py7D7gPoKKiwo0fPz7l+TVr1mSlV9DO3NtoZ4+7sLCQkSNHtnNELVu8eDEN98fOQHHnTmeLOZt3H70BDDSzAWaWD5wLLEiewMxGAvcCpzrntmYxFhERaYWsJQXnXATcylCGAAAPaklEQVSYDjwPrAEec86tMrObzexUb7JfAiXAPDNbbmYLmliciIjkQFavKTjnngGeaVB2Y9Lwidlcv4iItE3X6+YiCyZMmNDoi2h33HEHl19+ebPzlZSUALBp0ya/a4qGxo8fz9KlS5tdzh133EF1dbU/fuaZZ/r9GYmItEWXTApz3p5D+R3lBP4zQPkd5cx5e996SJ08eTJz585NKZs7dy6TJ09u1fx9+vThz3/+816vv2FSePzxx+nRo8deLy9botFoe4cgIi3ocklhzttzmPa/09iwcwMOx4adG5j2v9P2KTGcddZZPP300/43edevX8+mTZsYN24clZWVTJw4kVGjRjF06FDmz5/faP7169czZMgQAGpqajj33HMZNGgQp59+OjU1Nf50l19+ud/t9k033QTAb3/7WzZt2sSECROYMGECAEOGDGHbtm0A3H777X6314lut9evX8+gQYO49NJLOeqoozj55JNT1pMwb948hgwZwvDhw/na174GxA/s11xzDUOGDGHYsGF+V9oLFy5k5MiRDB06lIsvvpg6r4PB8vJyrrvuOkaNGsW8efNYt24dkyZNYvTo0YwbN4533313r+tdRDKvQ3xPIZOufO5Kln/WdNfZr218jbpo6p2v1eFqLpl/CfcvS+06OxqNEgwGGfGlEdwxqemO9nr27MmYMWN49tlnOe2005g7dy7f+c53MDMKCwt58skn6d69O9u2bePoo4/m1FNPbbLjubvvvpvi4mLWrFnDypUrU7q+njVrFj179iQajTJx4kRWrlzJD3/4Q26//XYWLVrEgQcemLKsZcuW8dBDD7FkyRKcc4wdO5YTTjiBAw44gA8++IBHH32U+++/n+985zs8/vjjnH/++Snz33zzzTz//PP07dvXb4667777WL9+PcuXLycUCrFjxw5qa2uZOnUqCxcu5PDDD+fCCy/k7rvv5sorrwTiP0L05ptvAjBx4kTuueceBg4cyJIlS/i3f/s3/vKXvzRZtyKSW13uTKFhQmipvLWSm5CSm46cc9xwww0MGzaME088kU8//ZQtW7Y0uZyXX37ZPzgPGzaMYcOG+c899thjjBo1ipEjR7Jq1aq0nd0le+WVVzj99NPp1q0bJSUlnHHGGX631wMGDGDEiBFA091zH3fccUydOpX777/fb/p56aWXuOyyy/xvSffs2ZP33nuPAQMGcPjhhwPw3e9+l5dfftlfzjnnxLuzqqys5NVXX+Xss89mxIgRXHbZZWzevLnZbRCR3NrvzhSa+0QPUH5HORt2Nu46u39ZfxZPXZxS1pYvgZ122mlcddVVvPnmm1RXVzN69GgA5syZw+eff86yZcvIy8ujvLw8bXfZLfnoo4/41a9+xRtvvMEBBxzA1KlT92o5CYlutyH+Leh0zUf33HMPS5Ys4emnn2b06NEsW7Zsr9aV6GI7FovRo0ePtL8EJyIdQ5c7U5g1cRbFealdZxfnFTNr4r51nV1SUsKECRO4+OKLUy4w79y5k4MPPpi8vDwWLVrEhnS/5ZDka1/7Go888ggA77zzDitXrgTi3W5369aNsrIytmzZwrPP7ukmqrS0NG1XFuPGjeOpp56iurqaqqoqnnzyybTdXjdl3bp1jB07lptvvpmDDjqITz75hJNOOol7773X//2GHTt2cMQRR7B+/Xq/59SHH36YE05o3GNJ9+7dGTBgAPPmzQPiZ1ErVqxodTwikn1dLilMGTqF+751H/3L+mMY/cv6c9+37mPK0H3vOnvy5MmsWLEiJSlMmTKFpUuXMnToUGbPns2RRx7Z7DIuv/xyKisrGTRoEDfeeKN/xjF8+HBGjhzJkUceyXnnnZfS7fa0adOYNGmSf6E5YdSoUUydOpUxY8YwduxYvve977Wp64hrr72WoUOHMmTIEI499liGDx/O9773Pfr16+f/jvMjjzxCYWEhDz30EGeffTZDhw4lEAjw/e9/P+0y58yZwwMPPOD/Kly6C+8i0n4s3qNq51FRUeEa3re/Zs0aBg0alPF17Q99CHUmyXFn6zXNtM7Wr02C4s6djhKzmS1zzlW0NF2XO1MQEZGmKSmIiIhPSUFERHz7TVLobNdGpGl6LUXaz36RFAoLC9m+fbsOJvsB5xzbt2+nsLCwvUMR6ZL2iy+vHXrooWzcuJHPP/88o8utra3tlAenzh53YWEhhx56aHuHI9Il7RdJIS8vjwEDBmR8uYsXL+4UPwnZkOIWkb21XzQfiYhIZigpiIiIT0lBRER8na6bCzP7HGi+V7nMORDYlqN1ZZLizp3OGDMo7lzqKDH3d84d1NJEnS4p5JKZLW1NXyEdjeLOnc4YMyjuXOpsMav5SEREfEoKIiLiU1Jo3n3tHcBeUty50xljBsWdS50qZl1TEBERn84URETE12WTgpkdZmaLzGy1ma0ys3/3yn9iZp+a2XLv8Y2kea43s7Vm9p6Zfb0dY19vZm978S31ynqa2Ytm9oH3/wCv3Mzst17cK81sVDvFfERSnS43s11mdmVHrG8ze9DMtprZO0llba5fM/uuN/0HZvbddoj5l2b2rhfXk2bWwysvN7OapDq/J2me0d6+tdbbLmuHuNu8T5jZJK9srZnNyGbMzcT9p6SY15vZcq+8w9R3qzjnuuQDOAQY5Q2XAu8Dg4GfANekmX4wsAIoAAYA64BgO8W+HjiwQdltwAxveAbwC2/4G8CzgAFHA0s6QN0Hgc+A/h2xvoGvAaOAd/a2foGewIfe/wO84QNyHPPJQMgb/kVSzOXJ0zVYzuvedpi3Xae0Q123aZ/wHuuAfwHyvWkG5zruBs//Grixo9V3ax5d9kzBObfZOfemN7wbWAP0bWaW04C5zrk659xHwFpgTPYjbbXTgD94w38Avp1UPtvFvQb0MLND2iPAJBOBdc655r6E2G717Zx7GdiRJp621O/XgRedczucc/8EXgQm5TJm59wLzrmIN/oa0GzXs17c3Z1zr7n4EWs2e7YzK5qo66Y0tU+MAdY65z50ztUDc71ps6a5uL1P+98BHm1uGe1R363RZZNCMjMrB0YCS7yi6d4p94OJZgLiCeOTpNk20nwSySYHvGBmy8xsmlfW2zm32Rv+DOjtDXekuBPOJfUN09HrG9pevx0t/ouJfxJNGGBmb5nZX81snFfWl3icCe0Zc1v2iY5W1+OALc65D5LKOnp9+7p8UjCzEuBx4Ern3C7gbuDLwAhgM/HTwI7meOfcKOAU4Aoz+1ryk96njg55W5mZ5QOnAvO8os5Q3yk6cv2mY2YzgQgwxyvaDPRzzo0ErgYeMbPu7RVfGp1un2hgMqkfejp6fafo0knBzPKIJ4Q5zrknAJxzW5xzUedcDLifPU0WnwKHJc1+qFeWc865T73/W4Enice4JdEs5P3f6k3eYeL2nAK86ZzbAp2jvj1trd8OEb+ZTQW+CUzxkhle88t2b3gZ8fb4w734kpuY2iXmvdgnOkRdA5hZCDgD+FOirKPXd0NdNil47X4PAGucc7cnlSe3t58OJO4uWACca2YFZjYAGEj8IlFOmVk3MytNDBO/mPiOF1/iDpfvAvO94QXAhd5dMkcDO5OaQdpDyqeojl7fSdpav88DJ5vZAV7zx8leWc6Y2STgP4BTnXPVSeUHmVnQG/4X4nX7oRf3LjM72nt/XMie7cxl3G3dJ94ABprZAO9M9Fxv2vZwIvCuc85vFuro9d1Ie1/pbq8HcDzxJoCVwHLv8Q3gYeBtr3wBcEjSPDOJZ/n3aKe7BIjfYbHCe6wCZnrlvYCFwAfAS0BPr9yAu7y43wYq2rHOuwHbgbKksg5X38ST1mYgTLyd95K9qV/i7fhrvcdF7RDzWuJt7Yn9+x5v2jO9fWc58CbwraTlVBA/CK8Dfo/3Bdccx93mfcJ7777vPTezPfYRr/x/gO83mLbD1HdrHvpGs4iI+Lps85GIiDSmpCAiIj4lBRER8SkpiIiIT0lBRER8SgrS4ZhZr6QeJT9r0GNmfiuX8ZCZHdHCNFeY2ZTMRN0xmNkrZjaiveOQzku3pEqHZmY/ASqdc79qUG7E999YuwTWQZnZK8B059zy9o5FOiedKUinYWZfsfjvX8wh/mWgQ8zsPjNbavHfxLgxadpXzGyEmYXM7Aszu9XMVpjZP8zsYG+aW8zsyqTpbzWz1y3eL/+xXnk3M3vcW++fvXU1+iRuZl/1OjtbZmbPmllvM8vzxo/3pvmlmf2nN/yfZvaGmb1jZvd4SS4Rx+3eelabWYXFfwvhAy9BJuphlZnNNbM1ZvaYmRWliekUb3vftHhf/92S4lht8Q7nfpHRF0k6PSUF6WyOBH7jnBvs4n1AzXDOVQDDgZPMbHCaecqAvzrnhgP/IP5N43TMOTcGuBZIJJgfAJ855wYDPyXem27qTGYFwJ3Amc650cAfgZ8658LARcB9ZnYyMAG4xZvtTufcV4GhXnzJ3WrXeNv0APAU8H1vumnm/VAO8d8WuMM5NwioBS5rENPBxH/3YaKLd564Evh3M+tN/Nu/RznnhgE/b6IupItSUpDOZp1zbmnS+GQze5N49wGDiB8sG6pxziW6jV5G/EdP0nkizTTHE++fH+dcomuRhgYBRwEvWfzXtmbgddDmnFvpzT8fuNhLFAATzex14t2VnODNn5Dot+dt4G0X7yCulviPKyU6UPvIxX+/AeJJ6PgGMR1LvC5e9WKa4m3TDiAG3G9mpwNVTdSFdFGh9g5ApI38g5iZDQT+HRjjnPvCzP4IFKaZpz5pOErT+31dK6ZJx4CVzrlxTTw/BNgJJJqtion3czPKOfepmd3SIO5EHLGk4cR4Iq6GFwMbjhvwnHPugkbBmlUAJwFnA5cT76xPBNCZgnRu3YHdxHuaTPzaWab9nfivaGFmQ0l/JrIa6GtmY7zp8s3sKG/4HKAEGA/cZfF+9IuIH+C3WbzH2zP3Iq4BZvZVb/g84JUGz78KnOD1ypm4NjLQW19359z/AVeRpjlMujadKUhn9ibxA/K7wAbiB/BM+x0w28xWe+taTfxTv885V2dmZwG/9Q76QeDXZvY58esQ451zm8zsXuLXQy4xsz94y9rMnl/8a4s1wNXeRe+3gfsaxLTFzC4B/pR0G+8NQA3whHcdJED8R19EfLolVaQZFv/RlJBzrtZrrnoBGOj2/PZxe8T0FeDPzjl9H0EyTmcKIs0rARZ6ycGAy9ozIYhkm84URETEpwvNIiLiU1IQERGfkoKIiPiUFERExKekICIiPiUFERHx/X9UugrMcr5WswAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier(**best_parameters)  \n",
    "train(knn, X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00       484\n",
      "           1       1.00      0.96      0.98        95\n",
      "\n",
      "   micro avg       0.99      0.99      0.99       579\n",
      "   macro avg       1.00      0.98      0.99       579\n",
      "weighted avg       0.99      0.99      0.99       579\n",
      "\n",
      "Accuracy: 0.9930915371329879\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAF25JREFUeJzt3X20XXV95/H3xwCigqEYsJoEggJVKiqaAg5rKTMiRUaDyyqSES0OJa2VVkdK66w6QmnVqqMdO+JoKhRFBcGxrrTG4hOS0YomDkh5EBqRhwQcnoMIhAe/88felxwvyb4nl7vvPTe8X2vdxdn7/M7e3/3j5nzu3r9zfjtVhSRJW/KEmS5AkjTaDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KzSpIrkhw6QZs9ktyTZM40ldW7JNclOax9fGqSz850TXr8MCg0Jdo3svvaN+j/l+SsJDtN9X6q6jer6tsTtLmhqnaqqoenev/tm/SD7XHeleRfkrxkqvcjjRKDQlPp1VW1E/AiYDHw7vEN0pjtv3dfaI9zHnAhcP4M1zPlkmw30zVodMz2f7AaQVW1Hvgq8DyAJN9O8t4k3wXuBZ6VZG6SM5LcnGR9kr8avFSU5IQkVyX5eZIrk7yoXT94CebAJGuS3N2exXykXb8oSY292SV5ZpIVSe5IsjbJCQP7OTXJeUk+0+7riiSLhzzOh4DPAfOT7DawzVcluXTgjOP5A88tTPKlJLcmuT3Jx9r1z07yrXbdbUk+l2SXyfR/kqPa/d+d5CdJjhjfdwPH/tlxfXZ8khuAbyX5apITx237R0le2z5+TpKvt/16dZKjJ1OvRp9BoSmXZCFwJHDJwOo3AcuAnYHrgbOAh4C9gQOAw4Hfa1//euBU4M3AU4ElwO2b2dVHgY9W1VOBZwPnbaGkc4F1wDOB1wHvS/IfBp5f0rbZBVgBfGzI49yhrfF24M523QHAmcDvA08DPgmsSPLENgj/qT3+RcD8dr8AAd7f1vhcYGHbB1slyYHAZ4CT2+N5KXDdVmziZe3+fxs4B1g6sO39gD2BryR5CvB14PPA7sAxwMfbNtrGGBSaSl9OchfwHeAi4H0Dz51VVVe0f4XvShMk76iqX1TVLcDf0LzZQBMYH6yq1dVYW1XXb2Z/DwJ7J5lXVfdU1cXjG7ShdQjwZ1V1f1VdCnyK5g1+zHeqamU7pnE28IIJjvPo9jjvA04AXtceFzRh+Mmq+n5VPVxVnwY2AgcDB9IEwcntcd9fVd8BaI/x61W1sapuBT5C86a9tY4Hzmy39cuqWl9VP96K15/a1nYf8A/AC5Ps2T73RuBLVbUReBVwXVX9fVU9VFWXAP8beP0kataIMyg0lV5TVbtU1Z5V9Yftm82YGwce7wlsD9zcXp65i+Yv793b5xcCPxlif8cD+wI/TrI6yas20+aZwB1V9fOBddfT/DU/5mcDj+8FdkyyXZI3toPW9yT56kCb86pqF+DpwOXAi8cd20ljx9Ue28K2joXA9QOh8ogkT09ybnsZ7m7gszRjIFtr2L7bkkf+P7V99hU2BfhSmktt0BznQeOO843Arz+GfWtEOWCl6TI4TfGNNH9lz9vcm2b7/LMn3GDVvwFL28Hx1wJfTPK0cc1uAnZNsvNAWOwBrB9i+59j0xvj5p6/LckyYE2Sz1fVzW3t762q945v3346ao8k223muN9H00f7V9UdSV7DkJfAxunqu18ATx5Y3tyb+vjppM8BTkmyCtiRZvB+bD8XVdUrJlGjZhnPKDTt2jfUrwEfTvLUJE9oB3PHLrV8CviTJC9uPiSVvQcufzwiybFJdquqXwJ3tat/OW5fNwL/Arw/yY7twPLxNH+xT8WxXA1cAPxpu+rvgD9IclBb+1OS/MckOwM/AG4G/rpdv2OSQ9rX7QzcA2xIMp9mjGEyzgDekuTlbb/OT/Kc9rlLgWOSbN8O2L9uiO2tpDl7OI3m015j/ftPwL5J3tRub/skv5XkuZOsWyPMoNBMeTOwA3AlzUDwF4FnAFTV+cB7aQZKfw58mWZcY7wjgCuS3EMzsH3MuMtdY5bSDB7fRHPd/ZSq+sYUHsuHgGVJdq+qNTTjFh9rj2stcBxAOwbyapoB/BtoBtjf0G7jL2g+VryB5nLPlyZTSFX9AHgLzZjPBpqxorGQ/W80Zxt3tvv7/BDb29jWcthg+/bs7HCay1I30Vy++wDwxMnUrdEWb1wkSeriGYUkqZNBIUnqZFBIkjoZFJKkTrPuexTz5s2rRYsWzXQZkjSr/PCHP7ytqnabuOWjzbqgWLRoEWvWrJnpMiRpVkmyuWlwhuKlJ0lSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUqbegSHJmkluSXL6F55Pkb9Pcw/iytPdEliSNlj7PKM6imQZ6S14J7NP+LAP+V4+1SJImqbcv3FXVqiSLOpocBXymmnnOL06yS5JntDe12aL774drrpnCQiVpFtl1V5g3mZvkPgYz+c3s+fzqfZTXteseFRTt7SaXAcyb9yxWrZqW+iRppGzc2ATF0qXTu99ZMYVHVS0HlgPsu+/iOuCAGS5IkmbA9dfDHXdM/35n8lNP64GFA8sLGOKG95Kk6TWTQbECeHP76aeDgQ0TjU9IkqZfb5eekpwDHArMS7IOOAXYHqCqPgGsBI6kufn8vTQ3hJckjZg+P/XUOdzSftrpbX3tX5I0NfxmtiSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTr0GRZIjklydZG2Sd23m+T2SXJjkkiSXJTmyz3okSVuvt6BIMgc4HXglsB+wNMl+45q9Gzivqg4AjgE+3lc9kqTJ6fOM4kBgbVVdW1UPAOcCR41rU8BT28dzgZt6rEeSNAnb9bjt+cCNA8vrgIPGtTkV+FqSPwKeAhy2uQ0lWQYsA9h99z2mvFBJ0pbN9GD2UuCsqloAHAmcneRRNVXV8qpaXFWL587dbdqLlKTHsz6DYj2wcGB5Qbtu0PHAeQBV9T1gR2BejzVJkrZSn0GxGtgnyV5JdqAZrF4xrs0NwMsBkjyXJihu7bEmSdJW6i0oquoh4ETgAuAqmk83XZHktCRL2mYnASck+RFwDnBcVVVfNUmStl6fg9lU1Upg5bh17xl4fCVwSJ81SJIem5kezJYkjTiDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR12m7YhknmA3sOvqaqVvVRlCRpdAwVFEk+ALwBuBJ4uF1dQGdQJDkC+CgwB/hUVf31ZtocDZzabu9HVfWfhi1ektS/Yc8oXgP8RlVtHHbDSeYApwOvANYBq5OsqKorB9rsA/xX4JCqujPJ7sOXLkmaDsOOUVwLbL+V2z4QWFtV11bVA8C5wFHj2pwAnF5VdwJU1S1buQ9JUs+GPaO4F7g0yTeBR84qquqPO14zH7hxYHkdcNC4NvsCJPkuzeWpU6vqn4esSZI0DYYNihXtTx/73wc4FFgArEqyf1XdNdgoyTJgGcDuu+/RQxmSpC0ZKiiq6tNJdqA9AwCurqoHJ3jZemDhwPKCdt2gdcD32239NMk1NMGxetz+lwPLAfbdd3ENU7MkaWoMNUaR5FDg32gGpz8OXJPkpRO8bDWwT5K92pA5hkeflXyZ5myCJPNogujaYYuXJPVv2EtPHwYOr6qrAZLsC5wDvHhLL6iqh5KcCFxAM/5wZlVdkeQ0YE1VrWifOzzJ2MduT66q2yd/OJKkqTZsUGw/FhIAVXVNkgk/BVVVK4GV49a9Z+BxAe9sfyRJI2jYoFiT5FPAZ9vlNwJr+ilJkjRKhg2KtwJvA8Y+Dvt/aMYqJEnbuGE/9bQR+Ej7I0l6HOkMiiTnVdXRSf6VZi6mX1FVz++tMknSSJjojOLt7X9f1XchkqTR1Pk9iqq6uX14G3BjVV0PPBF4AXBTz7VJkkbAsJMCrgJ2bO9J8TXgTcBZfRUlSRodwwZFqupe4LXAx6vq9cBv9leWJGlUDB0USV5C8/2Jr7Tr5vRTkiRplAwbFO+gucHQP7TTcDwLuLC/siRJo2LY71FcBFw0sHwtm758J0nahk30PYr/UVXvSPKPbP57FEt6q0ySNBImOqM4u/3vf++7EEnSaOoMiqr6YftwDXBfVf0SIMkcmu9TSJK2ccMOZn8TePLA8pOAb0x9OZKkUTNsUOxYVfeMLbSPn9zRXpK0jRg2KH6R5EVjC0leDNzXT0mSpFEy7P0o3gGcn+QmIMCvA2/orSpJ0sgY9nsUq5M8B/iNdtXVVfVgf2VJkkbFUJeekjwZ+DPg7VV1ObAoiVOPS9LjwLBjFH8PPAC8pF1eD/xVLxVJkkbKsEHx7Kr6IPAgQDuTbHqrSpI0MoYNigeSPIl2Go8kzwY29laVJGlkDPupp1OAfwYWJvkccAhwXF9FSZJGx4RBkSTAj2luWnQwzSWnt1fVbT3XJkkaARMGRVVVkpVVtT+bblokSXqcGHaM4v8m+a1eK5EkjaRhxygOAo5Nch3wC5rLT1VVz++rMEnSaBg2KH671yokSSNrojvc7Qj8AbA38K/AGVX10HQUJkkaDRONUXwaWEwTEq8EPtx7RZKkkTLRpaf92k87keQM4Af9lyRJGiUTnVE8MkOsl5wk6fFpoqB4QZK725+fA88fe5zk7ok2nuSIJFcnWZvkXR3tfidJJVm8tQcgSepX56Wnqpoz2Q0nmQOcDrwCWAesTrKiqq4c125n4O3A9ye7L0lSf4b9wt1kHAisraprq+oB4FzgqM20+0vgA8D9PdYiSZqkPoNiPnDjwPK6dt0j2vtwL6yqzqlBkixLsibJmg0bbp36SiVJW9RnUHRK8gTgI8BJE7WtquVVtbiqFs+du1v/xUmSHtFnUKwHFg4sL2jXjdkZeB7w7XZqkIOBFQ5oS9Jo6TMoVgP7JNkryQ7AMcCKsSerakNVzauqRVW1CLgYWFJVa3qsSZK0lXoLivZ7FycCFwBXAedV1RVJTkuypK/9SpKm1rCTAk5KVa0EVo5b954ttD20z1okSZMzY4PZkqTZwaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSp16DIskRSa5OsjbJuzbz/DuTXJnksiTfTLJnn/VIkrZeb0GRZA5wOvBKYD9gaZL9xjW7BFhcVc8Hvgh8sK96JEmT0+cZxYHA2qq6tqoeAM4FjhpsUFUXVtW97eLFwIIe65EkTUKfQTEfuHFgeV27bkuOB766uSeSLEuyJsmaDRtuncISJUkTGYnB7CTHAouBD23u+apaXlWLq2rx3Lm7TW9xkvQ4t12P214PLBxYXtCu+xVJDgP+HHhZVW3ssR5J0iT0eUaxGtgnyV5JdgCOAVYMNkhyAPBJYElV3dJjLZKkSeotKKrqIeBE4ALgKuC8qroiyWlJlrTNPgTsBJyf5NIkK7awOUnSDOnz0hNVtRJYOW7dewYeH9bn/iVJj91IDGZLkkZXr2cUkqSptXEjXHPN9O7ToJCkWWLuXPjZz2DVqsm8euenTHa/BoUkzRK77AIHHTTZV8+ZM9lXOkYhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnq1GtQJDkiydVJ1iZ512aef2KSL7TPfz/Joj7rkSRtvd6CIskc4HTglcB+wNIk+41rdjxwZ1XtDfwN8IG+6pEkTU6fZxQHAmur6tqqegA4FzhqXJujgE+3j78IvDxJeqxJkrSVtutx2/OBGweW1wEHbalNVT2UZAPwNOC2wUZJlgHL2qUHFy/+tet6qXjW2TgXnrhhpqsYDfbFJvbFJvbFJncvmOwr+wyKKVNVy4HlAEnWVN25eIZLGglNX9xrX2BfDLIvNrEvNkmyZrKv7fPS03pg4cDygnbdZtsk2Q6YC9zeY02SpK3UZ1CsBvZJsleSHYBjgBXj2qwAfrd9/DrgW1VVPdYkSdpKvV16asccTgQuAOYAZ1bVFUlOA9ZU1QrgDODsJGuBO2jCZCLL+6p5FrIvNrEvNrEvNrEvNpl0X8Q/4CVJXfxmtiSpk0EhSeo0skHh9B+bDNEX70xyZZLLknwzyZ4zUed0mKgvBtr9TpJKss1+NHKYvkhydPu7cUWSz093jdNliH8jeyS5MMkl7b+TI2eizr4lOTPJLUku38LzSfK3bT9dluRFQ224qkbuh2bw+yfAs4AdgB8B+41r84fAJ9rHxwBfmOm6Z7Av/j3w5PbxWx/PfdG22xlYBVwMLJ7pumfw92If4BLg19rl3We67hnsi+XAW9vH+wHXzXTdPfXFS4EXAZdv4fkjga8CAQ4Gvj/Mdkf1jMLpPzaZsC+q6sKqurddvJjmOyvbomF+LwD+kmbesPuns7hpNkxfnACcXlV3AlTVLdNc43QZpi8KeGr7eC5w0zTWN22qahXNJ0i35CjgM9W4GNglyTMm2u6oBsXmpv+Yv6U2VfUQMDb9x7ZmmL4YdDzNXwzbogn7oj2VXlhVX5nOwmbAML8X+wL7JvlukouTHDFt1U2vYfriVODYJOuAlcAfTU9pI2dr30+AWTKFh4aT5FhgMfCyma5lJiR5AvAR4LgZLmVUbEdz+elQmrPMVUn2r6q7ZrSqmbEUOKuqPpzkJTTf33peVf1ypgubDUb1jMLpPzYZpi9Ichjw58CSqto4TbVNt4n6YmfgecC3k1xHcw12xTY6oD3M78U6YEVVPVhVPwWuoQmObc0wfXE8cB5AVX0P2BGYNy3VjZah3k/GG9WgcPqPTSbsiyQHAJ+kCYlt9To0TNAXVbWhquZV1aKqWkQzXrOkqiY9GdoIG+bfyJdpziZIMo/mUtS101nkNBmmL24AXg6Q5Lk0QXHrtFY5GlYAb24//XQwsKGqbp7oRSN56an6m/5j1hmyLz4E7ASc347n31BVS2as6J4M2RePC0P2xQXA4UmuBB4GTq6qbe6se8i+OAn4uyT/hWZg+7ht8Q/LJOfQ/HEwrx2POQXYHqCqPkEzPnMksBa4F3jLUNvdBvtKkjSFRvXSkyRpRBgUkqROBoUkqZNBIUnqZFBIkjoZFNI4SR5OcmmSy5P8Y5Jdpnj7xyX5WPv41CR/MpXbl6aaQSE92n1V9cKqeh7Nd3TeNtMFSTPJoJC6fY+BSdOSnJxkdTuX/18MrH9zu+5HSc5u1726vVfKJUm+keTpM1C/9JiN5DezpVGQZA7NtA9ntMuH08yVdCDNfP4rkryUZo6xdwP/rqpuS7Jru4nvAAdXVSX5PeBPab4hLM0qBoX0aE9KcinNmcRVwNfb9Ye3P5e0yzvRBMcLgPOr6jaAqhq7H8AC4AvtfP87AD+dnvKlqeWlJ+nR7quqFwJ70pw5jI1RBHh/O37xwqrau6rO6NjO/wQ+VlX7A79PMxGdNOsYFNIWtHcN/GPgpHYq+wuA/5xkJ4Ak85PsDnwLeH2Sp7Xrxy49zWXTFM6/izRLeelJ6lBVlyS5DFhaVWe3U1R/r52l9x7g2Ham0vcCFyV5mObS1HE0d1U7P8mdNGGy10wcg/RYOXusJKmTl54kSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLU6f8D1P76LMW31RkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluate(knn, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SVM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameter tuning with GridsearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 4 candidates, totalling 12 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best options\n",
      "=======================================\n",
      "C: 0.5\n",
      "loss: 'squared_hinge'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Done   6 out of  12 | elapsed:    1.4s remaining:    1.4s\n",
      "[Parallel(n_jobs=10)]: Done  12 out of  12 | elapsed:    1.5s finished\n"
     ]
    }
   ],
   "source": [
    "param_grid={'C':(0.1, 0.5), 'loss': ('hinge','squared_hinge')}\n",
    "best_parameters = hyper_parameter_tuning(param_grid , LinearSVC(random_state=0), 3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training with best parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xt8FPW9//HXJ5s7hARQUbmFelBBriGigggc1IN9tFovVRFaqReUas+x/uwp1nNsjy1IeyxHaz0qHmurTaW1aqVVi0rBSynIpQgCUlEBuQgSIBByz35/f8zssEk2yQaymwTez8djHjvzne/MfHaymc/Md2a/a845REREAFLaOgAREWk/lBRERCSgpCAiIgElBRERCSgpiIhIQElBREQCSgoiIhJQUhARkUDCkoKZ/cLMdpvZ+43MNzP7mZltMrM1ZlaQqFhERCQ+qQlc9y+BnwNPNzL/EqC/P5wDPOq/NumEE05w+fn5LYtk717Yvh2qqiA9HXr2hG7dml3s0KFDdOrUqWXbSkAcLdVs3EmKo1n14qjo3p3MU09t0xhaui9a7TOS5L9Jo3G3089GJI5W/588wjhaolViboU4Vq5cucc5d2KzFZ1zCRuAfOD9RuY9DkyKmt4InNLcOkeMGOFa5Ne/di472zk4PGRne+XNWLRoUcu2laA4WqrJuJMYR5NixFGTkZHcOFphX7TKZ6QN/iYx427Hn41IHK36P3kUcbTEUcfcSnEAK1wcx21zCez7yMzygT855wbFmPcnYLZz7h1/eiHwXefciqbWWVhY6FasaLJKXfn5sGVL/PVFAFJSwKzZoSYcJjU1tel60HTZrl1QW9swhlAI+vRpWL+x8XjrAQcPHSInJ6du+dq13plofenpUFAQe31NxRKrXjzrWLwYKioaxpGZSfHQoXSPnCHXP3ZFTzc1L95l//pXqKxsGEdGBpx3Xtzr279/P3l5eS2PKVK2cmXsOPr2hc2bG5Y3wsxWOucKm6uXyOajVmNm04BpAD169GDx4sVxLzt261YsRrkDtkyZ0uSyVTU1pKe2zi7q++tfH3EcLdVU3MmMoyntIY6mYth6zTX+hHduZlHj0dPV1dWkhUJY5B86Mr/+QSa6rN46Tt6xI3YctbXsjm4qjVreossiAxyOM/oVvG1HTdemp1OZklInzm5VVbHjqKpiX1VVw/cVvf3ItuuVxZxXb7x+bDkVFbHjqKggtHcvB/bvjzE38kYPL+nMGp0Xz7K5lZWx46ispGTPnobrqzcd2X6tc+wrKWl2m43FmNdYHFu38mYLjoVxi+dy4kgH2kPzUd++dS+7IkPfvs0u2qqXqkcRR0s1GXcS42hSe4ijFWI46s9IOOxcnz6x4+jTx7na2qMbampiDosWLWpY3lQcjawn5nC0MTcRx6JFi45sneFwy4fG4mjhZ/SoPyOt9L9CnM1HbflI6nzg6/5TSOcCJc65na2+lZkzITu7bll2tleeTIqj2ThqMzKSG0d72BdmMGtW7DhmzfKasY5mCIViD9CwbNYsikakkX8HpHwf8u+AohFpXhyNrSfWcLQxNxUHHNk642gKbDA0Ekd45o8Iu3DcA9Ci+g2GmT+KvT8S9DlNWPORmT0LjANOMLNtwPeBNADn3GPAK8AXgU1AGfCNhAQyebL3es89sHWr10Y7c+bh8mRRHM3GsXHKFAYmM452vC/aIo6iITDtUqPMb8nZkudNMwQai8TVa05yuCbnOeeoCddQVVtFdW011eFqbzxcTU24hupwNfN7b+ZHl0JFVBw3ftmx4ZQNnFKyHz7xV2iHtxPZVphwk2VhF8awBmX14w+Hw7zVbTmPfdkRac2PxLGi+0ou+CD+p4m279/Ovg/2xV2/vre6r+TRenE093c5Ggm90ZwILb7RfBQWL17MuHHjkrKt1qS4k6e1Yi5aW8Q9C+9ha8lW+uT2YeaEmUwefPT/8lW1VRyqOkRpVSkHqw5SVl1GaVUpy1ctp88ZfSirKuNg1UEOVR1i9l9nc6DyQIN1ZKVm8c/9/jk4cNfU1niv/kE8Mt5guraGGldTZ35NuOao35N4+ub2ZfMdm+Ouf0zdaJZjU/0D4ZRTpjCOcW0dVlI55/jl6l9y2yu3UV5TDsCWki3c+NKNLN22lIKTCyirLuNQtXdgP1R9iLIqbzpSXlZdxqEq77W8utx7rSmnvKa86YPw2vhiLK8pZ8OeDaSlpBFKCZGWkkZqSippKWl0Su8UjKempDYYous3NYRSQqSaN37X63c1GssPB/6QHl/oUafMzLCYt2Kbnm/+TV3D6lzhRHzjpcYbL5667KlG59W39cOt9OnfJ+768caxtWTrEa+zKUoK0iaK1hYx7Y/TKKsuA7wD4QMHH2DA2gGtcobckjiO5gy91tVSXFbM/or9lFSWUFJRwr6Kfeyv2M++in2UVJR48ypK2F+5nwMVB4LXksoSSipLYh64K2sr+fm7P29QnhHKIDM1k6y0LDJTM8kMHR7PzcglKy2LrNQsMtMy6ZTWiaxUfzo1k6z0w+Mln5TQb0A/r76//BW/vYKdpQ1v652acyqvXvcqDucdSB2HHydykJLi3ZpMISU40KZYCimWgmGYWTBdZ55/sI4eNzMeWvYQnx74tEEcvbv0ZtQJozh70NkADdrrmxuvX+acC5qRwmG/Ccm89xR53zsO7oi5P0b1HtWgvDGnFJ9Cv9794q4fa3ux4uiTe+SJpilKCsehRDVVgPePdrDqIMVlxewp28Oesj0Ulxezt3yv91q2l73le3nhgxeoqKn7LHpluJIb/nADj7z7CGkpaaSFvDPM9FC6dzYaSvPKo+ZlhDKCOsH8UJo37Y83VvbWlrf46d9+SmWt11obOUN/c/ObDDxxIPsr9nOg8gD7K/YH4yUVJRyo8l4jzTG81fQ+yU7LpnN6Z3LSc8jJyCEnPYdTO58ajP/f3/8v5nKGseSGJXRO70x2WjZZaVmkpnj/spEDbPR4rDKoe0YcsaxkGef2P7dO2awJs7jt5dsoqymrE/v9E+7nC92+EKwjen0W4xHKo3X/hffXOWEI4rjwflKKU8jJyGn1bUY454KrhtkTZnPry7c2jGPC/fTLi/8gvz20vUX167t/wv1Mf3l6gzhmTuhgN5qlfYp1hj7tj9MA6iSGsAtTUlHiHczLitlTvofisqiDe/neOsO+in3sK9/X6JlvRFZqFrmZuQ0SQkRVuIqacA1l1WWNtlVHblAmop26sraSJ1Y9EUynpaTROb0zXTK6eAf2jBz65PYh50TvgB7eF6Z3n950yehCTkaO95qeQ9fMruRl5dE1qyuZqZlBE0rkLDl6eO3j12I2BfTJ7cO5vc9ttfcWLcVSyE6r+8TT1GFTSQulJeyEIV6R7cWKoyXfUToS0U1NXxv6NVJSUo56fxhGWijtiGP6+tCvE0oJJe3voqRwHKgN11JcXsyu0l3cueDOOmccAGXVZdw0/ybm/G0O+8r3BWfGsdpZIzqldSI3M5fcjFxyM3M5retp5J7sjedl5pGXkUdeZl4w3SWjC3kZeWSkZpCSksLYX46NeUncM6cnz1/9fIPL/kYv9QEX9p5oqXHezc3qcHWdm6LV4epgXqRedbiaqX+YGvO9GcabU9+kS0YXMlMzg/buyNVJdDv4u399l1FjRjU40LfErAmzYp4ZJ+pMsCmTB09OehJQHO0rDiWFJIputumd25sfjf8RkwZPCi5Zoy9d65dFHp2LDGXVZew+tJvPD33O7rLd7Cnbw+eHPufzss/5ZNsn/OfH/1mn6abWxehGIUpFTQWZoUzOPOHM4EDfNbNr8Nolo8vhg31WHpmpmcGZb2pKKiELeQfPlNQ6bciRNuNIu3HkgPmTi37S4ECYkZLBjy/6Mb1ze8e1PyNPzkX2T2Q83nn/+Zf/bLTt+txe59aJuzEplkJmamZc8TamqTNjkWRTUkiSZ957hlv+dEvwhMnWkq1M+9M0dh/azZdO/xLOOQ5UHqC4wmt331O+h71lXlPNnrI9QbNNcVkxxeXFlFaVxtxOVmoWuaFcTgmdQu/c3gw9eSjdsroFw6y3Z1FcXtxguV5devGn6/4U90H9aMU6EE45ZUqLDoR12sqPoGm7sbbrWRfOOqrL/SPRXs5IRZQUEqiqtoqK6gpKKkuY8caMICFEVNRUcPfCu3lgyQMUlxdTHa5usA7D6JrVlROzT6R7dneGnjw0GD8x+0S6ZnalW1Y3rw07M4/s9Gw+Wf0J+UPzSUvxbq5mpGZ4N2RDqXTP7s43X/5mgwPh7Atn0zWra8L3SbT6B8JEtxfH2j7oDF0kmpJCK3LOUVlbSVl1Gfsr9vPZwc94d/u7LNuxjB2lDdvPAarD1YzNHxsc6E/IPqHO0C2rG4ZRE66h1tUG7emRxwMjT+BkpmYGT+lsD22nf/f+Mc/qrx96PakpqToQ+nSGLlKXksJRCrswFTUVlFaVsu3ANpZuW8qybctYtn0ZG4s3ApCTnkNmambMJ2565vTkpxf/lFpXS024Jrhn4JzDMCprKklNSSUzNZOM1IzgwB+yw+339UWafBqjA6GINEZJ4QjUhGuoqKlg58GdvL3lbZZuX8rSbUtZ9/k6wi5MZmomI3uO5IoBVzC692gG9xjM/I3z+ffX/71OE1JmaibfPvfblFWXkR5KD75YlB5KD27ahiyUkGfBRURiUVKIg3OOqtoq9lfsZ/Hmxby99W2WbV/Ge5+9R3W4mtSUVApOKeCOc+5gdJ/RDD95OBmpGVTXeo9FlleXc/FpF1NVW8Wcv81hx8Ed9OrSix+O/yFThkwhlBJq67coIgIoKTQq7MLUhGt4fdPrvPHJGyz5dAkrdq6goqYCwxjSYwg3F9zM6D6jGdlzJBmhDKpqq6gN1wa9PmaEMsjLzAvO/r997re587w72/qtiYg0SkkhSk1tDas+W8WCTQtYvGUxy7Yu49DbhwA4o/sZXDfoOs7vcz5n9zybTmmdvKeFnPflsGqrDrojiHS50FqPb4qIJMtxkRQa6+vHOceGPRt4/aPXWfjJQpZ8uiR4hr9vbl8uOPECvlTwJUaeOpK8rDzC4TAORyglREZqBt3SupEeSg/uAYiIdHTHfFKI1dfPDS95na5tKdkSdLXQo1MPxuaP5bxe5zHy1JH06NyDzas30+8L/chM9XucTMsKOlQTETkWHfNJ4Z6F9zTo66eqtoql25Yy8Z8mcuuIWxnZcyR9c/uSYimkhbw+4rPTstmRuoP+3frr6R8ROW4c80mhqR+iePiSh8lOyyY7LZuM1IzgR0QiEtU1sIhIe3XMJ4U+uX3YUrKlQXnv3N6c1u20NohIRKT9OuYfj5k5YWaDfuOz07KZNWFWG0UkItJ+HfNJYfLgycz98lz65vbFMPrm9mXul+eqmwcRkRiO+eYjUF8/IiLxOuavFEREJH5KCiIiElBSEBGRgJKCiIgElBRERCSgpCAiIgElBRERCSgpiIhIQElBREQCSgoiIhJQUhARkYCSgoiIBJQUREQkoKQgIiIBJQUREQkoKYiISEBJQUREAglNCmY20cw2mtkmM5sRY34fM1tkZn83szVm9sVExiMiIk1LWFIwsxDwCHAJMBCYZGYD61X7D+B3zrnhwLXA/yYqHhERaV4irxRGApuccx8756qAecBl9eo4oIs/ngvsSGA8IiLSjNQErrsn8GnU9DbgnHp1fgC8ZmbfAjoBFyYwHhERaYY55xKzYrOrgInOuZv86a8B5zjnbo+qc6cfw0/N7DzgSWCQcy5cb13TgGkAPXr0GDFv3ryExFxfaWkpnTt3Tsq2WpPiTp6OGDMo7mRqLzGPHz9+pXOusNmKzrmEDMB5wIKo6buBu+vVWQf0jpr+GDipqfWOGDHCJcuiRYuStq3WpLiTpyPG7JziTqb2EjOwwsVx7E7kPYXlQH8z62dm6Xg3kufXq7MVmABgZgOATODzBMYkIiJNSFhScM7VALcDC4ANeE8ZrTOz+8zsUr/a/wNuNrP3gGeBqX5GExGRNpDIG804514BXqlXdm/U+HpgdCJjEBGR+OkbzSIiElBSEBGRgJKCiIgElBRERCSgpCAiIgElBRERCSgpiIhIQElBREQCSgoiIhJQUhARkYCSgoiIBJQUREQkoKQgIiIBJQUREQkoKYiISEBJQUREAkoKIiISUFIQEZGAkoKIiASUFEREJKCkICIiASUFEREJKCmIiEhASUFERAJKCiIiElBSEBGRgJKCiIgElBRERCSgpCAiIgElBRERCSgpiIhIQElBREQCSgoiIhJQUhARkYCSgoiIBJQUREQkoKQgIiIBJQUREQkoKYiISCChScHMJprZRjPbZGYzGqlztZmtN7N1ZvabRMYjIiJNS03Uis0sBDwCXARsA5ab2Xzn3PqoOv2Bu4HRzrl9ZnZSouIREZHmxX2lYGbnm9k3/PETzaxfM4uMBDY55z52zlUB84DL6tW5GXjEObcPwDm3O/7QRUSktZlzrvlKZt8HCoEznHOnm9mpwHPOudFNLHMVMNE5d5M//TXgHOfc7VF1/gD8AxgNhIAfOOf+HGNd04BpAD169Bgxb968FrzFI1daWkrnzp2Tsq3WpLiTpyPGDIo7mdpLzOPHj1/pnCtsrl68zUeXA8OBVQDOuR1mlnMU8UVvvz8wDugFvGVmg51z+6MrOefmAnMBCgsL3bhx41ph081bvHgxydpWa1LcydMRYwbFnUwdLeZ4m4+qnHdJ4QDMrFMcy2wHekdN9/LLom0D5jvnqp1zn+BdNfSPMyYREWll8SaF35nZ40Cemd0MvAE80cwyy4H+ZtbPzNKBa4H59er8Ae8qATM7ATgd+DjOmEREpJXF1XzknHvAzC4CDgBnAPc6515vZpkaM7sdWIB3v+AXzrl1ZnYfsMI5N9+fd7GZrQdqge8454qP4v2IiMhRaDYp+I+WvuGcGw80mQjqc869ArxSr+zeqHEH3OkPIiLSxpptPnLO1QJhM8tNQjwiItKG4n36qBRYa2avA4cihc65f01IVCIi0ibiTQov+IOIiBzD4r3R/Cv/CaLT/aKNzrnqxIUlIiJtIa6kYGbjgF8BmwEDepvZ9c65txIXmoiIJFu8zUc/BS52zm0EMLPTgWeBEYkKTEREki/eL6+lRRICgHPuH0BaYkISEZG2Eu+Vwgoz+z/g1/70ZGBFYkISEZG2Em9SmA7cBkQeQX0b+N+ERCQiIm0m3qSQCjzknJsDwbecMxIWlYiItIl47yksBLKiprPwOsUTEZFjSLxJIdM5VxqZ8MezExOSiIi0lXiTwiEzK4hMmFkhUJ6YkEREpK3Ee0/hDuA5M9vhT58CXJOYkEREpK00eaVgZmeb2cnOueXAmcBvgWrgz8AnSYhPRESSqLnmo8eBKn/8POB7wCPAPvzfTBYRkWNHc81HIefcXn/8GmCuc+554HkzW53Y0EREJNmau1IImVkkcUwA/hI1L977ESIi0kE0d2B/FnjTzPbgPW30NoCZ/RNQkuDYREQkyZpMCs65mWa2EO9po9f831QG7wrjW4kOTkREkqvZJiDn3NIYZf9ITDgiItKW4v3ymoiIHAeUFEREJKCkICIiASUFEREJKCmIiEhASUFERAJKCiIiElBSEBGRgJKCiIgElBRERCSgpCAiIgElBRERCSgpiIhIQElBREQCSgoiIhJQUhARkYCSgoiIBBKaFMxsopltNLNNZjajiXpXmpkzs8JExiMiIk1LWFIwsxDwCHAJMBCYZGYDY9TLAf4NWJaoWEREJD6JvFIYCWxyzn3snKsC5gGXxaj3Q+DHQEUCYxERkTgkMin0BD6Nmt7mlwXMrADo7Zx7OYFxiIhInFLbasNmlgLMAabGUXcaMA2gR48eLF68OKGxRZSWliZtW61JcSdPR4wZFHcydbiYnXMJGYDzgAVR03cDd0dN5wJ7gM3+UAHsAAqbWu+IESNcsixatChp22pNijt5OmLMzinuZGovMQMrXBzH7kQ2Hy0H+ptZPzNLB64F5kcloxLn3AnOuXznXD6wFLjUObcigTGJiEgTEpYUnHM1wO3AAmAD8Dvn3Dozu8/MLk3UdkVE5Mgl9J6Cc+4V4JV6Zfc2UndcImMREZHm6RvNIiISUFIQEZGAkoKIiASUFEREJKCkICIiASUFEREJKCmIiEhASUFERAJKCiIiElBSEBGRgJKCiIgElBRERCSgpCAiIgElBRERCSgpiIhIQElBREQCSgoiIhJQUhARkYCSgoiIBJQUREQkoKQgIiIBJQUREQkoKYiISEBJQUREAkoKIiISUFIQEZGAkoKIiASUFEREJKCkICIigdS2DqA1VFdXs23bNioqKlp1vbm5uWzYsKFV15kMHT3uzMxMevXqRVpaWluHJHLcOSaSwrZt28jJySE/Px8za7X1Hjx4kJycnFZbX7J05Lg7d+5McXEx27Zto1+/fm0dkshx55hoPqqoqKB79+6tmhCkbZgZ3bt3b/WrPhGJzzGRFAAlhGOI/pYibeeYSQptqbi4mGHDhjFs2DBOPvlkevbsGUxXVVXFtY5vfOMbbNy4sck6jzzyCEVFRa0RsohITMfEPYUWKyqCe+6BrVuhTx+YORMmTz7i1XXv3p3Vq1cD8IMf/IDOnTtz11131anjnMM5R0pK7Dz81FNPNbud22677YhjTKTm3puIdBzH339xURFMmwZbtoBz3uu0aV55K9u0aRMDBw5k8uTJnHXWWezcuZNp06ZRWFjIWWedxX333RfUPf/881m9ejU1NTXk5eUxY8YMhg4dynnnncfu3bsB+I//+A8efPDBoP6MGTMYOXIkZ5xxBkuWLAHg0KFDTJkyhYEDB3LVVVdRWFgYJKxo3/nOdxg4cCBDhgzhu9/9LgCfffYZl112GUOGDGHo0KEsW7YMgJ/85CcMGjSIQYMG8fDDDzf63l599VXOO+88CgoKuOaaazh06FCr71MRSaxj70rhjjsgxkEwsHQpVFbWLSsrgxtvhCeeqFOcVVsLoRAMGwb+wbilPvjgA55++mkKCwsBmD17Nt26daOmpobx48dz1VVXMXDgwDrLlJSUMHbsWGbPns2dd97JL37xC2bMmNFg3c453n33XebPn899993Hn//8Zx5++GFOOukkXnrpJd577z0KCgoaLLdr1y5eeeUV1q1bh5mxf/9+wLsSueiii7j99tupqamhrKyMZcuWUVRUxPLly6mpqWHkyJGMGzeOrKysOu9t9+7dzJ49m4ULF5Kdnc3MmTN56KGH+N73vndE+01E2sbxd6VQPyE0V36UTjvttCAhADz77LMUFBRQUFDAhg0bWL9+fYNlsrKyuOSSSwAYMWIEmzdvjrnuK664okGdd955h6uuugqAoUOHctZZZzVYrlu3bqSkpHDzzTfz4osv0qlTJwAWL17MLbfcAkBqaipdunThnXfe4corryQrK4ucnBy+8pWv8Pbbbzd4b0uWLGH9+vWMGjWKYcOGUVRU1GjcItJ+HXtXCs2d0efne01G9fXtC4sX1ykqb4Xn/SMHXIAPP/yQhx56iHfffZe8vDymTJkS89HL9PT0YDwUClFTUxNz3RkZGc3WiSUtLY0VK1bw+uuv89xzz/Hoo4/y2muvAS178if6vTnnmDhxIs8880zcy4tI+5PQKwUzm2hmG81sk5k1aP8wszvNbL2ZrTGzhWbWN5HxAN5N5ezsumXZ2V55gh04cICcnBy6dOnCzp07WbBgQatvY/To0bzwwgsArF27NuaVyMGDBzlw4ABf+tKX+J//+R/+/ve/AzB+/Hgee+wxAGprazlw4ABjxozhxRdfpLy8nNLSUl566SXGjBnTYJ2jRo3izTff5OOPPwa8exsffvhhq78/EUmshF0pmFkIeAS4CNgGLDez+c656KPU34FC51yZmU0HfgJck6iYgMNPGbXi00fxKigoYODAgZx55pn07duX0aNHt/o2vvWtb3HdddcxcODAYMjNza1Tp6SkhCuuuILKykrC4TBz5swB4Oc//zk333wzjz/+OKmpqTz++OOMHDmSSZMmcfbZZwMwffp0Bg8ezKZNm+qss0ePHjz55JNcc801wWO4s2bNon///q3+HkUkgSKPE7b2AJwHLIiavhu4u4n6w4G/NrfeESNGuPrWr1/foKw1HDhwICHrTaTq6mq3e/du55xz//jHP1x+fr6rrq5u46jiE72/E/U3bW2LFi1q6xCOiOJOnvYSM7DCxXHsTuQ9hZ7Ap1HT24Bzmqh/I/BqAuM5LpSWlnLxxRcTDodxzgVn/SIi8WgXRwszmwIUAmMbmT8NmAZeM8XiejeEc3NzOXjwYKvHVVtbm5D1JlIoFGLRokWEQqGgrKO8h+j9XVFR0eDv3B6VlpZ2iDjrU9zJ09FiTmRS2A70jpru5ZfVYWYXAvcAY51zMZ8Ldc7NBeYCFBYWunHjxtWZv2HDhoT0CtqRexvt6HFnZmYyfPjwNo6oeYsXL6b+57EjUNzJ09FiTuTTR8uB/mbWz8zSgWuB+dEVzGw48DhwqXNudwJjERGROCQsKTjnaoDbgQXABuB3zrl1ZnafmV3qV/tvoDPwnJmtNrP5jaxORESSIKH3FJxzrwCv1Cu7N2r8wkRuX0REWub46+YiAcaPH9/gi2gPPvgg06dPb3K5zp07A7Bjx46ga4r6xo0bx4oVK5pcz4MPPkhZWVkwfeWVVwb9GYmItMRxmRSK1haR/2A+Kf+VQv6D+RStPboeUidNmsS8efPqlM2bN49JkybFtfypp57K73//+yPefv2k8Pzzz5OXl3fE60uU2tratg5BRJpx3CWForVFTPvjNLaUbMHh2FKyhWl/nHZUieGqq67i5ZdfDr7Ju3nzZnbs2MGYMWMoLS1lwoQJFBQUMHjwYF566aUGy2/evJlBgwYBUF5ezrXXXsuAAQO4/PLLKS8vD+pNnz496Hb7+9//PgA/+9nP2LFjB+PHj2f8+PEADBo0iD179gAwZ86coNvrSLfbmzdvZsCAAdx8882cddZZXHzxxXW2E/Hcc88xaNAghg4dygUXXAB4B/a77rqLQYMGMWTIkKAr7YULFzJ8+HAGDx7MDTfcQKXfwWB+fj7f/e6jbv7pAAANwUlEQVR3KSgo4LnnnuOjjz5i4sSJjBgxgjFjxvDBBx8c8X4XkdbXLr6n0Jru+PMdrP6s8a6zl25bSmVt3Sdfy6rLuPGlG3liZd2us2trawmFQgw7eRgPTmy8o71u3boxcuRIXn31VS677DLmzZvH1VdfjZmRmZnJiy++SJcuXdizZw/nnnsul156aaMdzz366KNkZ2ezYcMG1qxZU6fr65kzZ9KtWzdqa2uZMGECa9as4V//9V+ZM2cOixYt4oQTTqizrpUrV/LUU0+xbNkynHOcc845jB07lq5du/Lhhx/y7LPP8sQTT3D11Vfz/PPPM2XKlDrL33fffSxYsICePXsGzVFz585l8+bNrF69mtTUVPbu3UtFRQVTp05l4cKFnH766Xz961/n0Ucf5Y477gC8HyFatWoVABMmTOCxxx6jf//+LFu2jG9+85v85S9/aXTfikhyHXdXCvUTQnPl8YpuQopuOnLO8b3vfY8hQ4Zw4YUXsn37dnbt2tXoet56663g4DxkyBCGDBkSzPvd735HQUEBw4cPZ926dTE7u4v2zjvvcPnll9OpUyc6d+7MFVdcEXR73a9fP4YNGwY03j336NGjmTp1Kk888UTQ9PPGG29wyy23BN+S7tatGxs3bqRfv36cfvrpAFx//fW89dZbwXquucbrzqq0tJQlS5bw1a9+lWHDhnHLLbewc+fOJt+DiCTXMXel0NQZPUD+g/lsKWnYdXbf3L4snrq4TllLvgR22WWX8e1vf5tVq1ZRVlbGiBEjACgqKuLzzz9n5cqVpKWlkZ+fH7O77OZ88sknPPDAAyxfvpyuXbsyderUI1pPRKTbbfC+BR2r+eixxx5j2bJlvPzyy4wYMYKVK1ce0bYiXWyHw2Hy8vJi/hKciLQPx92VwswJM8lOq9t1dnZaNjMnHF3X2Z07d2b8+PHccMMNdW4wl5SUcNJJJ5GWlsaiRYvYEuu3HKJccMEF/OY3vwHg/fffZ82aNYDX7XanTp3Izc1l165dvPrq4W6icnJyYnZlMWbMGP7whz9QVlbGoUOHePHFF2N2e92Yjz76iHPOOYf77ruPE088kU8//ZSLLrqIxx9/PPj9hr1793LGGWewefPmoOfUZ555hrFjG/ZY0qVLF/r168dzzz0HeFdR7733XtzxiEjiHXdJYfLgycz98lz65vbFMPrm9mXul+cyefDRd509adIk3nvvvTpJYfLkyaxYsYLBgwfz9NNPc+aZZza5junTp1NaWsqAAQO49957gyuOoUOHMnz4cM4880yuu+66Ot1uT5s2jYkTJwY3miMKCgqYOnUqI0eO5JxzzuGmm25qUdcR3/nOdxg8eDCDBg1i1KhRDB06lJtuuok+ffoEv+P8m9/8hszMTJ566im++tWvMnjwYFJSUrj11ltjrrOoqIgnn3wy+FW4WDfeRaTtmNejasdRWFjo6j+3v2HDBgYMGNDq2zoW+hDqSKLjTtTftLV1tH5tIhR38rSXmM1spXOusLl6x92VgoiINE5JQUREAkoKIiISOGaSQke7NyKN099SpO0cE0khMzOT4uJiHUyOAc45iouLyczMbOtQRI5Lx8SX13r16sW2bdv4/PPPW3W9FRUVHfLg1NHjzszMpFevXm0djshx6ZhICmlpafTr16/V17t48eIO8ZOQ9SluETlSx0TzkYiItA4lBRERCSgpiIhIoMN1c2FmnwNN9yrXek4A9iRpW61JcSdPR4wZFHcytZeY+zrnTmyuUodLCslkZivi6SukvVHcydMRYwbFnUwdLWY1H4mISEBJQUREAkoKTZvb1gEcIcWdPB0xZlDcydShYtY9BRERCehKQUREAsdtUjCz3ma2yMzWm9k6M/s3v/wHZrbdzFb7wxejlrnbzDaZ2UYz+5c2jH2zma3141vhl3Uzs9fN7EP/tatfbmb2Mz/uNWZW0EYxnxG1T1eb2QEzu6M97m8z+4WZ7Taz96PKWrx/zex6v/6HZnZ9G8T832b2gR/Xi2aW55fnm1l51D5/LGqZEf5na5P/vqwN4m7xZ8LMJvplm8xsRiJjbiLu30bFvNnMVvvl7WZ/x8U5d1wOwClAgT+eA/wDGAj8ALgrRv2BwHtABtAP+AgItVHsm4ET6pX9BJjhj88AfuyPfxF4FTDgXGBZO9j3IeAzoG973N/ABUAB8P6R7l+gG/Cx/9rVH++a5JgvBlL98R9HxZwfXa/eet7134f57+uSNtjXLfpM+MNHwBeAdL/OwGTHXW/+T4F729v+jmc4bq8UnHM7nXOr/PGDwAagZxOLXAbMc85VOuc+ATYBIxMfadwuA37lj/8K+EpU+dPOsxTIM7NT2iLAKBOAj5xzTX0Jsc32t3PuLWBvjHhasn//BXjdObfXObcPeB2YmMyYnXOvOedq/MmlQJNdz/pxd3HOLXXeEetpDr/PhGhkXzemsc/ESGCTc+5j51wVMM+vmzBNxe2f7V8NPNvUOtpif8fjuE0K0cwsHxgOLPOLbvcvuX8RaSbASxifRi22jaaTSCI54DUzW2lm0/yyHs65nf74Z0APf7w9xR1xLXX/Ydr7/oaW79/2Fv8NeGeiEf3M7O9m9qaZjfHLeuLFGdGWMbfkM9He9vUYYJdz7sOosva+vwPHfVIws87A88AdzrkDwKPAacAwYCfeZWB7c75zrgC4BLjNzC6InumfdbTLx8rMLB24FHjOL+oI+7uO9rx/YzGze4AaoMgv2gn0cc4NB+4EfmNmXdoqvhg63GeinknUPelp7/u7juM6KZhZGl5CKHLOvQDgnNvlnKt1zoWBJzjcZLEd6B21eC+/LOmcc9v9193Ai3gx7oo0C/mvu/3q7SZu3yXAKufcLugY+9vX0v3bLuI3s6nAl4DJfjLDb34p9sdX4rXHn+7HF93E1CYxH8Fnol3sawAzSwWuAH4bKWvv+7u+4zYp+O1+TwIbnHNzosqj29svByJPF8wHrjWzDDPrB/THu0mUVGbWycxyIuN4NxPf9+OLPOFyPfCSPz4f+Lr/lMy5QElUM0hbqHMW1d73d5SW7t8FwMVm1tVv/rjYL0saM5sI/DtwqXOuLKr8RDML+eNfwNu3H/txHzCzc/3/j69z+H0mM+6WfiaWA/3NrJ9/JXqtX7ctXAh84JwLmoXa+/5uoK3vdLfVAJyP1wSwBljtD18EngHW+uXzgVOilrkHL8tvpI2eEsB7wuI9f1gH3OOXdwcWAh8CbwDd/HIDHvHjXgsUtuE+7wQUA7lRZe1uf+MlrZ1ANV47741Hsn/x2vE3+cM32iDmTXht7ZHP92N+3Sv9z85qYBXw5aj1FOIdhD8Cfo7/Bdckx93iz4T/v/sPf949bfEZ8ct/Cdxar2672d/xDPpGs4iIBI7b5iMREWlISUFERAJKCiIiElBSEBGRgJKCiIgElBSk3TGz7lE9Sn5Wr8fM9DjX8ZSZndFMndvMbHLrRN0+mNk7ZjasreOQjkuPpEq7ZmY/AEqdcw/UKze8z2+4TQJrp8zsHeB259zqto5FOiZdKUiHYWb/ZN7vXxThfRnoFDOba2YrzPtNjHuj6r5jZsPMLNXM9pvZbDN7z8z+ZmYn+XV+ZGZ3RNWfbWbvmtcv/yi/vJOZPe9v9/f+thqciZvZ2X5nZyvN7FUz62Fmaf70+X6d/zaz//LH/8vMlpvZ+2b2mJ/kInHM8bez3swKzfsthA/9BBnZD+vMbJ6ZbTCz35lZVoyYLvHf7yrz+vrvFBXHevM6nPtxq/6RpMNTUpCO5kzgf5xzA53XB9QM51whMBS4yMwGxlgmF3jTOTcU+BveN41jMefcSOA7QCTBfAv4zDk3EPghXm+6dRcyywAeAq50zo0Afg380DlXDXwDmGtmFwPjgR/5iz3knDsbGOzHF92tdrn/np4E/gDc6tebZv4P5eD9tsCDzrkBQAVwS72YTsL73YcJzus8cQ3wb2bWA+/bv2c554YA9zeyL+Q4paQgHc1HzrkVUdOTzGwVXvcBA/AOlvWVO+ci3UavxPvRk1heiFHnfLz++XHORboWqW8AcBbwhnm/tjUDv4M259waf/mXgBv8RAEwwczexeuuZKy/fESk3561wFrndRBXgffjSpEO1D5x3u83gJeEzq8X0yi8fbHEj2my/572AmHgCTO7HDjUyL6Q41RqWwcg0kLBQczM+gP/Box0zu03s18DmTGWqYoar6Xxz31lHHViMWCNc25MI/MHASVApNkqG6+fmwLn3HYz+1G9uCNxhKPGI9ORuOrfDKw/bcCfnXNfaxCsWSFwEfBVYDpeZ30igK4UpGPrAhzE62ky8mtnre2veL+ihZkNJvaVyHqgp5mN9Oulm9lZ/vg1QGdgHPCIef3oZ+Ed4PeY1+PtlUcQVz8zO9sfvw54p978JcBYv1fOyL2R/v72ujjn/gR8mxjNYXJ805WCdGSr8A7IHwBb8A7gre1h4GkzW+9vaz3eWX/AOVdpZlcBP/MP+iHgp2b2Od59iHHOuR1m9jje/ZAbzexX/rp2cvgX/1piA3Cnf9N7LTC3Xky7zOxG4LdRj/F+DygHXvDvg6Tg/eiLSECPpIo0wbwfTUl1zlX4zVWvAf3d4d8+bouY/gn4vXNO30eQVqcrBZGmdQYW+snBgFvaMiGIJJquFEREJKAbzSIiElBSEBGRgJKCiIgElBRERCSgpCAiIgElBRERCfx/RlJ1KyHz1bkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "svm = LinearSVC(random_state=0, **best_parameters)  \n",
    "train(svm, X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99       484\n",
      "           1       1.00      0.95      0.97        95\n",
      "\n",
      "   micro avg       0.99      0.99      0.99       579\n",
      "   macro avg       0.99      0.97      0.98       579\n",
      "weighted avg       0.99      0.99      0.99       579\n",
      "\n",
      "Accuracy: 0.9913644214162349\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAF21JREFUeJzt3X20XXV95/H3xwCigqEYsJoEgjxUKT6gKeC4ljIjUmQ0uKwiGdHiUNJaaXW0ts6qo5RWrTrasQOOpkJRVBAc60prKD6hGa1o4oAUUGhEHhJwAIEgAuHB7/yx9yXHS7LvyeXue07C+7XWXZy9z+/s/d0/bs7n7v0757dTVUiStCWPGXUBkqTxZlBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRTapiS5IsnhU7TZK8ldSebMUlm9S3JtkiPax6ck+fSoa9Kjh0GhGdG+kd3TvkH/vyRnJdllpvdTVb9ZVd+Yos31VbVLVT040/tv36Tvb4/zjiT/kuT5M70faZwYFJpJL6+qXYDnAouBd05ukMa2/nv3ufY45wEXAeePuJ4Zl2SHUdeg8bGt/4PVGKqq9cAFwEEASb6R5D1Jvg3cDTwtydwkZyS5Kcn6JH81eKkoyUlJfpjk50muTPLcdv3gJZhDkqxJcmd7FvPhdv2iJDXxZpfkqUlWJLktydokJw3s55Qk5yX5VLuvK5IsHvI4HwA+A8xPssfANl+W5NKBM45nDTy3MMkXktyS5GdJTmvX75vk6+26W5N8Jslu0+n/JMe0+78zyY+THDW57waO/dOT+uzEJNcDX09yQZKTJ237B0le2T5+epKvtP16VZJjp1Ovxp9BoRmXZCFwNHDJwOrXAcuAXYHrgLOAB4D9gIOBI4Hfa1//auAU4PXAE4ElwM82s6uPAB+pqicC+wLnbaGkc4F1wFOBVwHvTfIfBp5f0rbZDVgBnDbkce7U1vgz4PZ23cHAmcDvA08CPg6sSPLYNgj/qT3+RcD8dr8AAd7X1vgMYGHbB1slySHAp4C3t8fzQuDardjEi9r9/zZwDrB0YNsHAnsDX0ryBOArwGeBPYHjgI+2bbSdMSg0k76Y5A7gW8A3gfcOPHdWVV3R/hW+O02QvKWqflFVNwN/Q/NmA01gfKCqVldjbVVdt5n93Q/sl2ReVd1VVRdPbtCG1guAP6uqe6vqUuATNG/wE75VVSvbMY2zgWdPcZzHtsd5D3AS8Kr2uKAJw49X1Xer6sGq+iSwETgMOIQmCN7eHve9VfUtgPYYv1JVG6vqFuDDNG/aW+tE4Mx2W7+sqvVV9aOteP0pbW33AP8APCfJ3u1zrwW+UFUbgZcB11bV31fVA1V1CfC/gVdPo2aNOYNCM+kVVbVbVe1dVX/YvtlMuGHg8d7AjsBN7eWZO2j+8t6zfX4h8OMh9ncicADwoySrk7xsM22eCtxWVT8fWHcdzV/zE3468PhuYOckOyR5bTtofVeSCwbanFdVuwFPBi4Hnjfp2N42cVztsS1s61gIXDcQKg9J8uQk57aX4e4EPk0zBrK1hu27LXno/1PbZ19iU4AvpbnUBs1xHjrpOF8L/Poj2LfGlANWmi2D0xTfQPNX9rzNvWm2z+875Qar/g1Y2g6OvxL4fJInTWp2I7B7kl0HwmIvYP0Q2/8Mm94YN/f8rUmWAWuSfLaqbmprf09VvWdy+/bTUXsl2WEzx/1emj56ZlXdluQVDHkJbJKuvvsF8PiB5c29qU+eTvoc4N1JVgE70wzeT+znm1X1kmnUqG2MZxSade0b6peBDyV5YpLHtIO5E5daPgH8SZLnNR+Syn4Dlz8ekuT4JHtU1S+BO9rVv5y0rxuAfwHel2TndmD5RJq/2GfiWK4CLgT+tF31d8AfJDm0rf0JSf5jkl2B7wE3AX/drt85yQva1+0K3AVsSDKfZoxhOs4A3pDkxW2/zk/y9Pa5S4HjkuzYDti/aojtraQ5eziV5tNeE/37T8ABSV7Xbm/HJL+V5BnTrFtjzKDQqLwe2Am4kmYg+PPAUwCq6nzgPTQDpT8HvkgzrjHZUcAVSe6iGdg+btLlrglLaQaPb6S57v7uqvrqDB7LB4FlSfasqjU04xantce1FjgBoB0DeTnNAP71NAPsr2m38Rc0HyveQHO55wvTKaSqvge8gWbMZwPNWNFEyP43mrON29v9fXaI7W1sazlisH17dnYkzWWpG2ku370feOx06tZ4izcukiR18YxCktTJoJAkdTIoJEmdDApJUqdt7nsU8+bNq0WLFo26DEnapnz/+9+/tar2mLrlw21zQbFo0SLWrFkz6jIkaZuSZHPT4AzFS0+SpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqVNvQZHkzCQ3J7l8C88nyd+muYfxZWnviSxJGi99nlGcRTMN9Ja8FNi//VkG/K8ea5EkTVNvX7irqlVJFnU0OQb4VDXznF+cZLckT2lvarNF994LV189g4VK0hjbfXeYN52b4s6gUX4zez6/eh/lde26hwVFe7vJZQDz5j2NVatmpT5JGqmNG5ugWLp0tHVsE1N4VNVyYDnAAQcsroMPHnFBkjQLrrsObrtt1FWM9lNP64GFA8sLGOKG95Kk2TXKoFgBvL799NNhwIapxickSbOvt0tPSc4BDgfmJVkHvBvYEaCqPgasBI6mufn83TQ3hJckjZk+P/XUOfzSftrpTX3tX5I0M/xmtiSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTr0GRZKjklyVZG2Sd2zm+b2SXJTkkiSXJTm6z3okSVuvt6BIMgc4HXgpcCCwNMmBk5q9Ezivqg4GjgM+2lc9kqTp6fOM4hBgbVVdU1X3AecCx0xqU8AT28dzgRt7rEeSNA079Ljt+cANA8vrgEMntTkF+HKSPwKeAByxuQ0lWQYsA9hzz71mvFBJ0paNejB7KXBWVS0AjgbOTvKwmqpqeVUtrqrFc+fuMetFStKjWZ9BsR5YOLC8oF036ETgPICq+g6wMzCvx5okSVupz6BYDeyfZJ8kO9EMVq+Y1OZ64MUASZ5BExS39FiTJGkr9RYUVfUAcDJwIfBDmk83XZHk1CRL2mZvA05K8gPgHOCEqqq+apIkbb0+B7OpqpXAyknr3jXw+ErgBX3WIEl6ZEY9mC1JGnMGhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqtMOwDZPMB/YefE1VreqjKEnS+BgqKJK8H3gNcCXwYLu6gM6gSHIU8BFgDvCJqvrrzbQ5Fjil3d4Pquo/DVu8JKl/w55RvAL4jaraOOyGk8wBTgdeAqwDVidZUVVXDrTZH/ivwAuq6vYkew5fuiRpNgw7RnENsONWbvsQYG1VXVNV9wHnAsdManMScHpV3Q5QVTdv5T4kST0b9ozibuDSJF8DHjqrqKo/7njNfOCGgeV1wKGT2hwAkOTbNJenTqmqfx6yJknSLBg2KFa0P33sf3/gcGABsCrJM6vqjsFGSZYBywD23HOvHsqQJG3JUEFRVZ9MshPtGQBwVVXdP8XL1gMLB5YXtOsGrQO+227rJ0mupgmO1ZP2vxxYDnDAAYtrmJolSTNjqDGKJIcD/0YzOP1R4OokL5ziZauB/ZPs04bMcTz8rOSLNGcTJJlHE0TXDFu8JKl/w156+hBwZFVdBZDkAOAc4HlbekFVPZDkZOBCmvGHM6vqiiSnAmuqakX73JFJJj52+/aq+tn0D0eSNNOGDYodJ0ICoKquTjLlp6CqaiWwctK6dw08LuCt7Y8kaQwNGxRrknwC+HS7/FpgTT8lSZLGybBB8UbgTcDEx2H/D81YhSRpOzfsp542Ah9ufyRJjyKdQZHkvKo6Nsm/0szF9Cuq6lm9VSZJGgtTnVG8uf3vy/ouRJI0njq/R1FVN7UPbwVuqKrrgMcCzwZu7Lk2SdIYGHZSwFXAzu09Kb4MvA44q6+iJEnjY9igSFXdDbwS+GhVvRr4zf7KkiSNi6GDIsnzab4/8aV23Zx+SpIkjZNhg+ItNDcY+od2Go6nARf1V5YkaVwM+z2KbwLfHFi+hk1fvpMkbcem+h7F/6iqtyT5Rzb/PYolvVUmSRoLU51RnN3+97/3XYgkaTx1BkVVfb99uAa4p6p+CZBkDs33KSRJ27lhB7O/Bjx+YPlxwFdnvhxJ0rgZNih2rqq7Jhbax4/vaC9J2k4MGxS/SPLciYUkzwPu6ackSdI4GfZ+FG8Bzk9yIxDg14HX9FaVJGlsDPs9itVJng78Rrvqqqq6v7+yJEnjYqhLT0keD/wZ8OaquhxYlMSpxyXpUWDYMYq/B+4Dnt8urwf+qpeKJEljZdig2LeqPgDcD9DOJJveqpIkjY1hg+K+JI+jncYjyb7Axt6qkiSNjWE/9fRu4J+BhUk+A7wAOKGvoiRJ42PKoEgS4Ec0Ny06jOaS05ur6taea5MkjYEpg6KqKsnKqnomm25aJEl6lBh2jOL/JvmtXiuRJI2lYccoDgWOT3It8Auay09VVc/qqzBJ0ngYNih+u9cqJElja6o73O0M/AGwH/CvwBlV9cBsFCZJGg9TjVF8ElhMExIvBT7Ue0WSpLEy1aWnA9tPO5HkDOB7/ZckSRonU51RPDRDrJecJOnRaaqgeHaSO9ufnwPPmnic5M6pNp7kqCRXJVmb5B0d7X4nSSVZvLUHIEnqV+elp6qaM90NJ5kDnA68BFgHrE6yoqqunNRuV+DNwHenuy9JUn+G/cLddBwCrK2qa6rqPuBc4JjNtPtL4P3AvT3WIkmapj6DYj5ww8DyunbdQ9r7cC+sqs6pQZIsS7ImyZoNG26Z+UolSVvUZ1B0SvIY4MPA26ZqW1XLq2pxVS2eO3eP/ouTJD2kz6BYDywcWF7QrpuwK3AQ8I12apDDgBUOaEvSeOkzKFYD+yfZJ8lOwHHAioknq2pDVc2rqkVVtQi4GFhSVWt6rEmStJV6C4r2excnAxcCPwTOq6orkpyaZElf+5UkzaxhJwWclqpaCayctO5dW2h7eJ+1SJKmZ2SD2ZKkbYNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE69BkWSo5JclWRtknds5vm3JrkyyWVJvpZk7z7rkSRtvd6CIskc4HTgpcCBwNIkB05qdgmwuKqeBXwe+EBf9UiSpqfPM4pDgLVVdU1V3QecCxwz2KCqLqqqu9vFi4EFPdYjSZqGPoNiPnDDwPK6dt2WnAhcsLknkixLsibJmg0bbpnBEiVJUxmLwewkxwOLgQ9u7vmqWl5Vi6tq8dy5e8xucZL0KLdDj9teDywcWF7QrvsVSY4A/hx4UVVt7LEeSdI09HlGsRrYP8k+SXYCjgNWDDZIcjDwcWBJVd3cYy2SpGnqLSiq6gHgZOBC4IfAeVV1RZJTkyxpm30Q2AU4P8mlSVZsYXOSpBHp89ITVbUSWDlp3bsGHh/R5/4lSY/cWAxmS5LGV69nFJKkR2bjRrj66tHWYFBI0piaOxd++lNYtWomtrbrE6b7SoNCksbUbrvBoYfO1NbmzJnuKx2jkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdeo1KJIcleSqJGuTvGMzzz82yefa57+bZFGf9UiStl5vQZFkDnA68FLgQGBpkgMnNTsRuL2q9gP+Bnh/X/VIkqanzzOKQ4C1VXVNVd0HnAscM6nNMcAn28efB16cJD3WJEnaSjv0uO35wA0Dy+uAQ7fUpqoeSLIBeBJw62CjJMuAZe3S/YsX/9q1vVS8zdk4Fx67YdRVjAf7YhP7YhP7YpM7F0z3lX0GxYypquXAcoAka6puXzziksZC0xd32xfYF4Psi03si02SrJnua/u89LQeWDiwvKBdt9k2SXYA5gI/67EmSdJW6jMoVgP7J9knyU7AccCKSW1WAL/bPn4V8PWqqh5rkiRtpd4uPbVjDicDFwJzgDOr6ookpwJrqmoFcAZwdpK1wG00YTKV5X3VvA2yLzaxLzaxLzaxLzaZdl/EP+AlSV38ZrYkqZNBIUnqNLZB4fQfmwzRF29NcmWSy5J8Lcneo6hzNkzVFwPtfidJJdluPxo5TF8kObb93bgiyWdnu8bZMsS/kb2SXJTkkvbfydGjqLNvSc5McnOSy7fwfJL8bdtPlyV57lAbrqqx+6EZ/P4x8DRgJ+AHwIGT2vwh8LH28XHA50Zd9wj74t8Dj28fv/HR3Bdtu12BVcDFwOJR1z3C34v9gUuAX2uX9xx13SPsi+XAG9vHBwLXjrrunvrihcBzgcu38PzRwAVAgMOA7w6z3XE9o3D6j02m7Iuquqiq7m4XL6b5zsr2aJjfC4C/pJk37N7ZLG6WDdMXJwGnV9XtAFV18yzXOFuG6YsCntg+ngvcOIv1zZqqWkXzCdItOQb4VDUuBnZL8pSptjuuQbG56T/mb6lNVT0ATEz/sb0Zpi8GnUjzF8P2aMq+aE+lF1bVl2azsBEY5vfiAOCAJN9OcnGSo2atutk1TF+cAhyfZB2wEvij2Slt7Gzt+wmwjUzhoeEkOR5YDLxo1LWMQpLHAB8GThhxKeNiB5rLT4fTnGWuSvLMqrpjpFWNxlLgrKr6UJLn03x/66Cq+uWoC9sWjOsZhdN/bDJMX5DkCODPgSVVtXGWapttU/XFrsBBwDeSXEtzDXbFdjqgPczvxTpgRVXdX1U/Aa6mCY7tzTB9cSJwHkBVfQfYGZg3K9WNl6HeTyYb16Bw+o9NpuyLJAcDH6cJie31OjRM0RdVtaGq5lXVoqpaRDNes6Sqpj0Z2hgb5t/IF2nOJkgyj+ZS1DWzWeQsGaYvrgdeDJDkGTRBccusVjkeVgCvbz/9dBiwoapumupFY3npqfqb/mObM2RffBDYBTi/Hc+/vqqWjKzongzZF48KQ/bFhcCRSa4EHgTeXlXb3Vn3kH3xNuDvkvwXmoHtE7bHPyyTnEPzx8G8djzm3cCOAFX1MZrxmaOBtcDdwBuG2u522FeSpBk0rpeeJEljwqCQJHUyKCRJnQwKSVIng0KS1MmgkCZJ8mCSS5NcnuQfk+w2w9s/Iclp7eNTkvzJTG5fmmkGhfRw91TVc6rqIJrv6Lxp1AVJo2RQSN2+w8CkaUnenmR1O5f/Xwysf3277gdJzm7Xvby9V8olSb6a5MkjqF96xMbym9nSOEgyh2bahzPa5SNp5ko6hGY+/xVJXkgzx9g7gX9XVbcm2b3dxLeAw6qqkvwe8Kc03xCWtikGhfRwj0tyKc2ZxA+Br7Trj2x/LmmXd6EJjmcD51fVrQBVNXE/gAXA59r5/ncCfjI75Uszy0tP0sPdU1XPAfamOXOYGKMI8L52/OI5VbVfVZ3RsZ3/CZxWVc8Efp9mIjppm2NQSFvQ3jXwj4G3tVPZXwj85yS7ACSZn2RP4OvAq5M8qV0/celpLpumcP5dpG2Ul56kDlV1SZLLgKVVdXY7RfV32ll67wKOb2cqfQ/wzSQP0lyaOoHmrmrnJ7mdJkz2GcUxSI+Us8dKkjp56UmS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmd/j+6jfos6eS6oAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluate(svm, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import gc\n",
    "import time\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Input, GRU, LSTM, Flatten, BatchNormalization, Bidirectional\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras import backend as K\n",
    "from tensorflow import set_random_seed\n",
    "from keras.callbacks import EarlyStopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_random_seed(7) # Define the seed in order to have reproducible data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_up(model):\n",
    "    K.clear_session()\n",
    "    del model\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MLP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_up(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/spiros/env/lib/python3.5/site-packages/ipykernel_launcher.py:6: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(2000, kernel_initializer=\"glorot_normal\", activation=\"relu\", input_shape=(1000,))`\n",
      "  \n",
      "/home/spiros/env/lib/python3.5/site-packages/ipykernel_launcher.py:8: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1000, kernel_initializer=\"glorot_normal\", activation=\"relu\")`\n",
      "  \n",
      "/home/spiros/env/lib/python3.5/site-packages/ipykernel_launcher.py:10: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(500, kernel_initializer=\"glorot_normal\", activation=\"relu\")`\n",
      "  # Remove the CWD from sys.path while we load stuff.\n",
      "/home/spiros/env/lib/python3.5/site-packages/ipykernel_launcher.py:12: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(100, kernel_initializer=\"glorot_normal\", activation=\"relu\")`\n",
      "  if sys.path[0] == '':\n",
      "/home/spiros/env/lib/python3.5/site-packages/ipykernel_launcher.py:14: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(10, kernel_initializer=\"glorot_normal\", activation=\"relu\")`\n",
      "  \n",
      "/home/spiros/env/lib/python3.5/site-packages/ipykernel_launcher.py:16: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1, kernel_initializer=\"glorot_normal\", activation=\"sigmoid\")`\n",
      "  app.launch_new_instance()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 2000)              2002000   \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 2000)              0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1000)              2001000   \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 1000)              0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 500)               500500    \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 500)               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 100)               50100     \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 10)                1010      \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 10)                0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 4,554,621\n",
      "Trainable params: 4,554,621\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 2082 samples, validate on 232 samples\n",
      "Epoch 1/10\n",
      "2082/2082 [==============================] - 2s 799us/step - loss: 0.2448 - acc: 0.8794 - val_loss: 0.0238 - val_acc: 0.9914\n",
      "Epoch 2/10\n",
      "2082/2082 [==============================] - 1s 484us/step - loss: 0.0400 - acc: 0.9765 - val_loss: 0.0014 - val_acc: 1.0000\n",
      "Epoch 3/10\n",
      "2082/2082 [==============================] - 1s 485us/step - loss: 0.0243 - acc: 0.9808 - val_loss: 3.2135e-06 - val_acc: 1.0000\n",
      "Epoch 4/10\n",
      "2082/2082 [==============================] - 1s 480us/step - loss: 0.0270 - acc: 0.9841 - val_loss: 3.4632e-04 - val_acc: 1.0000\n",
      "Epoch 5/10\n",
      "2082/2082 [==============================] - 1s 485us/step - loss: 0.0170 - acc: 0.9875 - val_loss: 3.6697e-06 - val_acc: 1.0000\n",
      "Epoch 6/10\n",
      "2082/2082 [==============================] - 1s 485us/step - loss: 0.0290 - acc: 0.9851 - val_loss: 3.8538e-07 - val_acc: 1.0000\n",
      "Epoch 7/10\n",
      "2082/2082 [==============================] - 1s 488us/step - loss: 0.0175 - acc: 0.9890 - val_loss: 2.5072e-06 - val_acc: 1.0000\n",
      "Epoch 8/10\n",
      "2082/2082 [==============================] - 1s 486us/step - loss: 0.0136 - acc: 0.9899 - val_loss: 9.7380e-07 - val_acc: 1.0000\n",
      "Epoch 9/10\n",
      "2082/2082 [==============================] - 1s 485us/step - loss: 0.0169 - acc: 0.9914 - val_loss: 3.3867e-07 - val_acc: 1.0000\n",
      "Epoch 10/10\n",
      "2082/2082 [==============================] - 1s 509us/step - loss: 0.0162 - acc: 0.9885 - val_loss: 1.5192e-07 - val_acc: 1.0000\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "\n",
    "input_shape = X_train.shape[1]\n",
    "model = Sequential()\n",
    "model.add(Dense(2000, input_shape=(input_shape,), init='glorot_normal', activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(1000, init='glorot_normal', activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(500, init='glorot_normal', activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(100, init='glorot_normal', activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(10, init='glorot_normal', activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(1, init='glorot_normal', activation='sigmoid'))\n",
    "model.summary()\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "history = model.fit(X_train, y_train, epochs=10, batch_size=16, verbose=1, validation_split=0.1)\n",
    "\n",
    "y_pred = model.predict(X_test)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "579/579 [==============================] - 0s 32us/step\n",
      "LOSS ON TEST DATA=0.06561413904697223\n",
      "ACCURACY ON TEST DATA=0.9930915371329879\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl4nOV97//3V6PVWkaSJYOtMZYDNt4kb8KEgA0ulEIWKImb4EIKNAlXOIckTX7tOXS5Qg49+R2appQkza8nJIU0SYMPgZC4LTk0i9mSBmyz2NjGC7bB8ip5kWRr13x/f8wjeSzLlmRr9Gg0n9d1zaWZZ5n5asDPR/d9P8/9mLsjIiJyNllhFyAiImOfwkJERAalsBARkUEpLEREZFAKCxERGZTCQkREBqWwEDkPZlZtZm5m2UPY9k4ze+l830ckDAoLyRhmttvMOs2sot/y14IDdXU4lYmMfQoLyTS7gJW9L8ysBpgQXjki6UFhIZnm+8AfJb2+A/he8gZmFjWz75lZg5m9Y2Z/ZWZZwbqImX3VzBrNbCfwgQH2/Scz229me83sf5pZZLhFmtkUM1ttZkfMbIeZfSpp3RIzW2dmzWZ20MweCpbnm9kPzOywmR0zs7VmdsFwP1tkIAoLyTS/BUrMbHZwEL8V+EG/bb4BRIH3AFeTCJe7gnWfAj4ILATqgBX99v0u0A1cEmxzPfDJc6hzFVAPTAk+4/81s98J1n0N+Jq7lwAXA08Ey+8I6p4KTAQ+DbSdw2eLnEZhIZmot3Xxu8AWYG/viqQA+XN3b3H33cDfAR8PNvko8LC773H3I8D/Str3AuD9wJ+4+wl3PwT8ffB+Q2ZmU4Ergf/u7u3u/jrwHU62iLqAS8yswt2Pu/tvk5ZPBC5x9x53X+/uzcP5bJEzUVhIJvo+8IfAnfTrggIqgBzgnaRl7wBVwfMpwJ5+63pNC/bdH3QDHQO+BUwaZn1TgCPu3nKGGj4BzATeCrqaPpj0ez0LrDKzfWb2FTPLGeZniwxIYSEZx93fITHQ/X7gx/1WN5L4C31a0rKLONn62E+imyd5Xa89QAdQ4e6lwaPE3ecOs8R9QLmZFQ9Ug7tvd/eVJELob4AnzazQ3bvc/X+4+xzgfSS6y/4IkRGgsJBM9Qngd9z9RPJCd+8hMQbwZTMrNrNpwBc4Oa7xBPBZM4uZWRlwX9K++4H/AP7OzErMLMvMLjazq4dTmLvvAX4D/K9g0Lo2qPcHAGZ2u5lVunscOBbsFjez5WZWE3SlNZMIvfhwPlvkTBQWkpHc/W13X3eG1Z8BTgA7gZeAHwKPBuu+TaKr5w3gVU5vmfwRkAtsBo4CTwKTz6HElUA1iVbG08D97v6LYN0NwCYzO05isPtWd28DLgw+r5nEWMzzJLqmRM6b6eZHIiIyGLUsRERkUAoLEREZlMJCREQGpbAQEZFBjZvpkCsqKry6ujrsMkRE0sr69esb3b1ysO3GTVhUV1ezbt2ZzoQUEZGBmNk7g2+lbigRERmClIaFmd1gZluDKZbvG2D9F8xss5ltMLNfBlfL9q7rMbPXg8fqVNYpIiJnl7JuqGDKgW+SmNmzHlhrZqvdfXPSZq8Bde7eamb3AF8BPhasa3P3BamqT0REhi6VYxZLgB3uvhPAzFYBN5OYBgEAd1+TtP1vgdtHsoCuri7q6+tpb28fybfNePn5+cRiMXJyNKGpSKZIZVhUcepUzvXA5WfZ/hPAz5Je55vZOhI3knnQ3X/Sfwczuxu4G+Ciiy7qv5r6+nqKi4uprq7GzIb/G8hp3J3Dhw9TX1/P9OnTwy5HREbJmBjgNrPbSdx17G+TFk9z9zoS9x142Mwu7r+fuz/i7nXuXldZefqZX+3t7UycOFFBMYLMjIkTJ6q1JpJhUhkWezl13v8YSXck62Vm1wF/Cdzk7h29y929d+7+ncBzJG5ROWwKipGn71Qk86QyLNYCM8xsupnlkri15ClnNZnZQhJ3ErspuAVl7/IyM8sLnleQuMVk8sD4iOnuiXOwuZ3Wzu5UvL2IyLiQsrBw927gXhJz/28BnnD3TWb2gJndFGz2t0AR8KN+p8jOBtaZ2RvAGhJjFikJCzM42NxOS/vIh8Xhw4dZsGABCxYs4MILL6SqqqrvdWdn55De46677mLr1q0jXpuIyHCk9Apud38GeKbfsi8mPb/uDPv9BqhJZW29IllZ5GVHaOvsGfH3njhxIq+//joAX/rSlygqKuJP//RPT9nG3XF3srIGzu3HHntsxOsSERmuMTHAHbaC3AhtXSMfFmeyY8cO5syZw2233cbcuXPZv38/d999N3V1dcydO5cHHnigb9urrrqK119/ne7ubkpLS7nvvvuYP38+V1xxBYcOHTrLp4iIjJxxMzfUYP7Hv25i877mAdd19cTp7I4zIS+b4QzdzplSwv0fmntO9bz11lt873vfo66uDoAHH3yQ8vJyuru7Wb58OStWrGDOnDmn7NPU1MTVV1/Ngw8+yBe+8AUeffRR7rvvtAvjRURGnFoWQCQrERHx+OjdYvbiiy/uCwqAxx9/nEWLFrFo0SK2bNnC5s2nD9EUFBRw4403ArB48WJ27949WuWKSIbLmJbF2VoAPXFn874mKovzuTCaPyr1FBYW9j3fvn07X/va13jllVcoLS3l9ttvH/A6htzc3L7nkUiE7m6dwSUio0MtCxIti7yc0R23SNbc3ExxcTElJSXs37+fZ599NpQ6RETOJGNaFoMpyInQ0t6Nu4/6RWeLFi1izpw5zJo1i2nTpnHllVeO6ueLiAzG3Eevnz6V6urqvP/Nj7Zs2cLs2bOHtP/h4x3sPdbGrAuLyc2OpKLEcWU4362IjF1mtj6YWums1A0VKMhNBEQqrrcQEUl3CotAfk4EM6M1pHELEZGxTGERyDIjPztLLQsRkQEoLJL0Xsk9XsZxRERGisIiyYTcCD1xp7M7HnYpIiJjisIiSUFOMMitcQsRkVMoLJLk5UTIMqN1BMctli9fftpFdg8//DD33HPPGfcpKioCYN++faxYsWLAba655hr6nyrc38MPP0xra2vf6/e///0cO3ZsqKWLiPRRWCTJMiM/Z2SnK1+5ciWrVq06ZdmqVatYuXLloPtOmTKFJ5988pw/u39YPPPMM5SWlp7z+4lI5lJY9DPSg9wrVqzg3//93/tudrR792727dvHwoULufbaa1m0aBE1NTX89Kc/PW3f3bt3M2/ePADa2tq49dZbmT17NrfccgttbW19291zzz1905vff//9AHz9619n3759LF++nOXLlwNQXV1NY2MjAA899BDz5s1j3rx5PPzww32fN3v2bD71qU8xd+5crr/++lM+R0QyV+ZM9/Gz++DAxkE3mxSPE+2K47mRwaf9uLAGbnzwrJuUl5ezZMkSfvazn3HzzTezatUqPvrRj1JQUMDTTz9NSUkJjY2NvPe97+Wmm24642f+4z/+IxMmTGDLli1s2LCBRYsW9a378pe/THl5OT09PVx77bVs2LCBz372szz00EOsWbOGioqKU95r/fr1PPbYY7z88su4O5dffjlXX301ZWVlbN++nccff5xvf/vbfPSjH+Wpp57i9ttvH/R7E5HxTS2LfiLBwbpnBE+fTe6K6u2Ccnf+4i/+gtraWq677jr27t3LwYMHz/geL7zwQt9Bu7a2ltra2r51TzzxBIsWLWLhwoVs2rRpwOnNk7300kvccsstFBYWUlRUxIc//GFefPFFAKZPn86CBQsATYMuIidlTstikBZAL3Nn975mygpzqSotGJGPvvnmm/n85z/Pq6++SmtrK4sXL+a73/0uDQ0NrF+/npycHKqrqweclnwwu3bt4qtf/Spr166lrKyMO++885zep1deXl7f80gkom4oEQHUsjiNmVEwwoPcRUVFLF++nD/+4z/uG9huampi0qRJ5OTksGbNGt55552zvseyZcv44Q9/CMCbb77Jhg0bgMT05oWFhUSjUQ4ePMjPfvazvn2Ki4tpaWk57b2WLl3KT37yE1pbWzlx4gRPP/00S5cuHalfV0TGocxpWQxDQW6EIyc6ibuTNULTla9cuZJbbrmlrzvqtttu40Mf+hA1NTXU1dUxa9ass+5/zz33cNdddzF79mxmz57N4sWLAZg/fz4LFy5k1qxZTJ069ZTpze+++25uuOEGpkyZwpo1a/qWL1q0iDvvvJMlS5YA8MlPfpKFCxeqy0lEzkhTlA/gWGsn7x5pZcakIgpylacD0RTlIuODpig/D7qSW0TkVAqLAeRmZxHJGtkruUVE0tm4D4tz6WZLxSD3eDJeui5FZOjGdVjk5+dz+PDhczq4FeRGaO+OE4/rwJjM3Tl8+DD5+flhlyIio2hcj97GYjHq6+tpaGgY9r5tnT0cPtFJ/EgeudnjOlOHLT8/n1gsFnYZIjKKxnVY5OTkMH369HPat/5oKx/+mzX89c1z+fgV1SNbmIhImtGfzGdQVVpAeWEuG+qbwi5FRCR0CoszMDNqY1GFhYgICouzqq2Ksv1QC62d3WGXIiISqpSGhZndYGZbzWyHmd03wPovmNlmM9tgZr80s2lJ6+4ws+3B445U1nkmNbFS4g6b9zWH8fEiImNGysLCzCLAN4EbgTnASjOb02+z14A6d68FngS+EuxbDtwPXA4sAe43s7JU1XomtbEoAG+oK0pEMlwqWxZLgB3uvtPdO4FVwM3JG7j7Gnfvve/nb4He8zF/D/i5ux9x96PAz4EbUljrgC4oyeeCkjw21uu+1SKS2VIZFlXAnqTX9cGyM/kE0Du/9pD2NbO7zWydma07l2sphqKmqpQNe9WyEJHMNiYGuM3sdqAO+Nvh7Ofuj7h7nbvXVVZWpqS2+bEoOxtO0NzelZL3FxFJB6kMi73A1KTXsWDZKczsOuAvgZvcvWM4+46GmmDc4k21LkQkg6UyLNYCM8xsupnlArcCq5M3MLOFwLdIBMWhpFXPAtebWVkwsH19sGzU1cZKAdioQW4RyWApm+7D3bvN7F4SB/kI8Ki7bzKzB4B17r6aRLdTEfAjS9yR7l13v8ndj5jZX5MIHIAH3P1Iqmo9m/LCXGJlBbo4T0QyWkrnhnL3Z4Bn+i37YtLz686y76PAo6mrbuhqY1E27NUZUSKSucbEAPdYVxsrZc+RNo6e6Ay7FBGRUCgshqC2KjHIrVNoRSRTKSyGYG4QFro4T0QylcJiCKIFObynolCD3CKSsRQWQ1Sj6cpFJIMpLIaopirKgeZ2DjW3h12KiMioU1gM0fypwcV5GuQWkQyksBiiOZNLyDJNVy4imUlhMUSFedlcMqlIZ0SJSEZSWAxDbayUjXubcPewSxERGVUKi2GojUVpPN7JviYNcotIZlFYDEONLs4TkQylsBiG2ZNLyM4yXW8hIhlHYTEM+TkRLr2wWGEhIhlHYTFMtbEoG+qPaZBbRDKKwmKYamOlNLd38+6R1rBLEREZNQqLYeod5NbFeSKSSRQWw3TphcXkZmfpjCgRySgKi2HKiWQxZ3KJBrlFJKMoLM5BbSzKm3ub6IlrkFtEMoPC4hzUVEU50dnDrsbjYZciIjIqFBbnoHe6cnVFiUimUFicg4sriyjIiSgsRCRjKCzOQSTLmFdVwgadESUiGUJhcY5qY6Vs2tdMd0887FJERFJOYXGOamNROrrjbDuoQW4RGf8UFueob7ryveqKEpHxT2FxjqonFlKcn61BbhHJCAqLc5SVZdRURRUWIpIRFBbnoSYW5a0DzXR094RdiohISikszsP8WCldPc7WAy1hlyIiklIKi/Og6cpFJFOkNCzM7AYz22pmO8zsvgHWLzOzV82s28xW9FvXY2avB4/VqazzXMXKCiibkKPpykVk3MtO1RubWQT4JvC7QD2w1sxWu/vmpM3eBe4E/nSAt2hz9wWpqm8kmBm1sVINcovIuJfKlsUSYIe773T3TmAVcHPyBu6+2903AGl7GXRtLMr2Q8dp69Qgt4iMX6kMiypgT9Lr+mDZUOWb2Toz+62Z/f5AG5jZ3cE26xoaGs6n1nNWUxWlJ+5s3q/WhYiMX2N5gHuau9cBfwg8bGYX99/A3R9x9zp3r6usrBz9CtF05SKSGVIZFnuBqUmvY8GyIXH3vcHPncBzwMKRLG6kXFCSz6TiPIWFiIxrqQyLtcAMM5tuZrnArcCQzmoyszIzywueVwBXApvPvld4amNRTVcuIuNaysLC3buBe4FngS3AE+6+ycweMLObAMzsMjOrB/4A+JaZbQp2nw2sM7M3gDXAg/3OohpTamOl7Gw8QUt7V9iliIikRMpOnQVw92eAZ/ot+2LS87Ukuqf67/cboCaVtY2kmlgUd3hzbzNXXDwx7HJEREbcWB7gThu1mq5cRMY5hcUImFiUR1VpgQa5RWTcUliMkMQgt8JCRMYnhcUIqYlFefdIK8daO8MuRURkxCksRsj8WOLivI171boQkfFHYTFC5k1JDHKrK0pExiOFxQiJTsiheuIEXZwnIuOSwmIE1cZK2aiWhYiMQ0MKCzO7OGn6jWvM7LNmVpra0tJPbSzKvqZ2Glo6wi5FRGREDbVl8RTQY2aXAI+QmCDwhymrKk3V6OI8ERmnhhoW8WCup1uAb7j7nwGTU1dWeppXFcVMg9wiMv4MNSy6zGwlcAfwb8GynNSUlL4K87K5pLJIYSEi485Qw+Iu4Argy+6+y8ymA99PXVnpqya4ktvdwy5FRGTEDCks3H2zu3/W3R83szKg2N3/JsW1paX5sVIaj3dwoLk97FJEREbMUM+Ges7MSsysHHgV+LaZPZTa0tJTTSwxyP3GHnVFicj4MdRuqKi7NwMfBr7n7pcD16WurPQ1Z3IJ2VmmM6JEZFwZalhkm9lk4KOcHOCWAeTnRJh5QbEGuUVkXBlqWDxA4vaob7v7WjN7D7A9dWWlt9pYlI17NcgtIuPHUAe4f+Tute5+T/B6p7t/JLWlpa+aWJRjrV3sOdIWdikiIiNiqAPcMTN72swOBY+nzOy0e2dLQu905Rs0biEi48RQu6EeA1YDU4LHvwbLZAAzLygmN5KlcQsRGTeGGhaV7v6Yu3cHj+8ClSmsK63lZmcxe0qJpisXkXFjqGFx2MxuN7NI8LgdOJzKwtJdbVWUN/c2E49rkFtE0t9Qw+KPSZw2ewDYD6wA7kxRTeNCTSzK8Y5udjaeCLsUEZHzNtSzod5x95vcvdLdJ7n77wM6G+osTt6TW11RIpL+zudOeV8YsSrGoYsrCynIiWiQW0TGhfMJCxuxKsah7EgWc6eUKCxEZFw4n7DQyO0gamOlbNrXRHdPPOxSRETOy1nDwsxazKx5gEcLiest5CxqY1Hau+LsaDgedikiIucl+2wr3b14tAoZj3qnK9+wp4lZF5aEXI2IyLk7n24oGcT0iYUU52Vr2g8RSXspDQszu8HMtprZDjO7b4D1y8zsVTPrNrMV/dbdYWbbg8cdqawzVbKyjHlVUTZqkFtE0lzKwsLMIsA3gRuBOcBKM5vTb7N3SVzc98N++5YD9wOXA0uA+4Pbuaad2liULftb6OzWILeIpK9UtiyWADuC6cw7gVXAzckbuPtud98A9D+S/h7wc3c/4u5HgZ8DN6Sw1pSpjZXS2RNn64GWsEsRETlnqQyLKmBP0uv6YNmI7Wtmd5vZOjNb19DQcM6FplJt7yC3xi1EJI2l9QC3uz/i7nXuXldZOTYnwY2VFVA2IYcNezRuISLpK5VhsReYmvQ6FixL9b5jiplREytlw16FhYikr1SGxVpghplNN7Nc4FYSN1AaimeB682sLBjYvj5YlpZqq6JsO9hCe1dP2KWIiJyTlIWFu3cD95I4yG8BnnD3TWb2gJndBGBml5lZPfAHwLfMbFOw7xHgr0kEzlrggWBZWqqJRemJO5v2NYddiojIOTnrFdzny92fAZ7pt+yLSc/XkuhiGmjfR4FHU1nfaOmbrrz+GIunpeUZwCKS4dJ6gDtdXFCSR2VxnsYtRCRtKSxGgZlRWxXVdOUikrYUFqOkNlbK2w3HOd7RHXYpIiLDprAYJbWxKO6wSV1RIpKGFBajpG+6cnVFiUgaUliMkoqiPKpKCzTILSJpSWEximqqomys1xxRIpJ+FBajqCYWZffhVppau8IuRURkWBQWo6jv4jx1RYlImlFYjKKaKk1XLiLpSWExiqITcpg2cYKmKxeRtKOwGGW1sVJ1Q4lI2lFYjLLaqih7j7XReLwj7FJERIZMYTHKei/O26iL80QkjSgsRtm8qihmupJbRNKLwmKUFeVlc3FlERt1RpSIpBGFRQhqq6K8Ud+Eu4ddiojIkCgsQlAbi9LQ0sHBZg1yi0h6UFiEoCa4knuD5okSkTShsAjBnMklRLJMg9wikjYUFiEoyI0w84JiTVcuImlDYRGS2mC6cg1yi0g6UFiEpCYW5WhrF/VH28IuRURkUAqLkMzvG+RWV5SIjH0Ki5DMvLCI3EiWzogSkbSgsAhJXnaEWZOL1bIQkbSgsAhRbSzKm3ubiMc1yC0iY5vCIkS1VaW0dHSz6/CJsEsRETkrhUWINF25iKQLhUWIZkwqIj8nS+MWIjLmKSxClB3JYu6UqM6IEpExL6VhYWY3mNlWM9thZvcNsD7PzP5PsP5lM6sOllebWZuZvR48/ncq6wxTTVWUTfua6e6Jh12KiMgZpSwszCwCfBO4EZgDrDSzOf02+wRw1N0vAf4e+JukdW+7+4Lg8elU1Rm2+VOjtHX18HaDBrlFZOxKZctiCbDD3Xe6eyewCri53zY3A/8cPH8SuNbMLIU1jTk1VYkrud9QV5SIjGGpDIsqYE/S6/pg2YDbuHs30ARMDNZNN7PXzOx5M1uawjpD9Z6KQorysnVGlIiMadlhF3AG+4GL3P2wmS0GfmJmc929OXkjM7sbuBvgoosuCqHM85eVZcyrKtF05SIypqWyZbEXmJr0OhYsG3AbM8sGosBhd+9w98MA7r4eeBuY2f8D3P0Rd69z97rKysoU/AqjozZWypZ9zXR2a5BbRMamVIbFWmCGmU03s1zgVmB1v21WA3cEz1cAv3J3N7PKYIAcM3sPMAPYmcJaQ1VTFaWzJ862gy1hlyIiMqCUhUUwBnEv8CywBXjC3TeZ2QNmdlOw2T8BE81sB/AFoPf02mXABjN7ncTA96fd/Uiqag2bpisXkbEupWMW7v4M8Ey/ZV9Met4O/MEA+z0FPJXK2saSqeUFlE7IYUP9Mf7w8vQcexGR8U1XcI8BZkZNVVQtCxEZsxQWY0RtLMq2gy20d/WEXYqIyGkUFmPEgqlldMedP/z2b1n1yru0tHeFXZKISB+FxRhx7axJ/NUHZtPU1sV9P97IZV/+BX+y6jVe2t5Ij26OJCIhM/fxcSCqq6vzdevWhV3GeXN33qhv4sn1e1j9+j6a27uZEs3nw4tifGRxjOkVhWGXKCLjiJmtd/e6QbdTWIxd7V09/GLLQZ5aX8/z2xqIOyyeVsaKxTE+UDuZkvycsEsUkTSnsBhnDja385PX9vKj9fXsOHScvOwsbph3ISsWx3jfxRVEsjJq/kURGSEKi3HK3dlQ38ST6+tZ/cY+mtq6uLAknw8vquIji2NcXFkUdokikkYUFhmgvauHX245xFOv1vPc1kPEHRZdVMqKxVP5QO1kogXqphKRs1NYZJhDze385PW9/GhdPduDbqrfm3shH1kc46pL1E0lIgNTWGQod2fj3kQ31U9fP9lNdcuiKj6yKMYlk9RNlW7cnZ2NJ6goylNrUUacwkLo6O7hV1sO8eT6ep7b1kBP3Fl4USkrFsf4YO0UHXjGsOb2Ln69vZHntjbw/LYGDjS3k2WwYGopS2dUsmxmBfNjpWRHdKmUnB+FhZziUEs7P31tHz9av4dtB4+Tm53F9XMuYMXiGEtnVI5qN1VHdw/Nbd00tXXR1NZFc/Cz/+uC3AiLp5WxZHo5k6MFo1ZfGNydLftbeG7bIZ7b2sCr7xylO+4U52ezdEYFV11SyYGmNl7Y3sgb9cdwh+K8bN53yUSWzqjk6pmVTC2fEPavIWlIYSEDcnfe3NvMk+v38NM39nGstYsLSvK4ZWGMFYuruGRS8ZDeo70rfvLg3t5FU+upB/zeg35z++nL27vOfpOnCbkRogU5tLR3c7yjG4BYWQFLqsu5bHo5l1WXc3FlIel+u/amti5e2t7I89sO8fy2Bg42dwAwZ3IJ11xayTWXTmLhRaXk9Gs9HGvt5Nc7DvPi9gZe2NbAvqZ2AKonTmDpjEqWzqjgiosnUqzrcGQIFBbDcXQ3lFWPZDlpoaO7hzVvJbqp1mxNdFPNn1rKdbMm0dkTH/Dg39TWTXNbF509Zz/gF+dlU1KQQzR4lBRk9z0/uezUn9GCHEryc8jNThwcu3vivHWghVd2HWHt7sSj8XgnABMLc6mrLmPJ9IksqS5n9uTiMd8l4+5s2tfM89saeG7rIV599xg9Qeth2YxKrr60kmtmVjKpJH9Y77mz8QQvbmvgxe2N/OfOw7R29hDJMhZdVNoXHrWxUp3kIANSWAzVsT3wjcUwfSlcez9Mrh354tJAQ0sHPw3Optp6sAUzKMnPGfAAP9CBv/dAHy3IoTg/OyUH7t4D49pdR3glCI89R9oAKMyNsGhaWV/rY8HUUvJzIiNew3A1tXbx4o6GvrGHhpZE62HulKTWw9SRG3vo6O7h1XeO8eL2RHi8ua8Jd4gW5HDVJRUsnVHB0pmVVJWO7249GTqFxVB1tcEr34YX/w7aj0HNH8Dyv4Ty6SNfZBpwd1o7eyjIiZCVBn+JHmhqTwRH0Pp460Di1rS5kSxqYlEuqy7n8unlLJpWNioD+vG4s3l/M89tTYw9vLYn0XqIFuSwdEYF11w6iWUzK5hUPPTWw/k4fLyDX799uK/lcaA50WX1nspClgWtjve+ZyKFeSm9D5qMYQqL4Wo7Br9+GH77vyHeDXV3wbL/BkWVI1ekpNyx1k7W7T7K2t2J1sfG+ia6444ZzLqwhCXVZVw2vZwl1eXD6u45m6bWLl7YfrL10Hg80XqoqYoGrYfKMXHmkruz49Bxng+C4+Vdh2nvipMTMRZPK0ucZTWjkrn/voYDAAALf0lEQVRTStLiDwUZGQqLc9W8H55/EF79PmTnw/s+A++7F/IGH/iVsaets4fX9hxl7a5EgKx/5yhtwQ2mpk2cwGXV5X1dV9UTJwxp0DweT4w9PLf1EM9ta+C1d48SD7p6ls1MjDssm1lJZXFeqn+989Le1cP6d47ywvYGXtzWyOb9zQCUTcjhqqDVsXRGxbg/Ey3TKSzOV+N2+NVfw+afwoQKWPZnidZG9tg+AMjZdfXE2bSvuW/cY93uIxxtTdxoqrI4LxEcQetj1oUlfYPCx1o7eWF7I89tPcQL2xr7Wg+1sSjXzKzk6ksnsWBqeg8iN7R08OsdjYnw2N7YN74y84KivoHyy6dPpCA3/LEgGTkKi5FSvx5+cT/sfhFKL4Llf5UY18ga22feyNDE487bDceTxj2OsvdYYtC8OD+bxdPKaG7r4vU9x4g7lE7IYdmMRNfSspmVVBSNzz8e3J23DrT0DZS/vOsInd1xsrOMC0ryqSjOo7Ioj8ripEfwelJxHhVFeeMmVNydlo5ujp7o5PCJzlN+nujsITvLyI4YOVlZRLKMnIiRHcnqW56dlZVYlpVFJNguO5LYLpKV2C4nkhVsm9g3J/jZ937Be6TidHGFxUhyh7d/Cb/4EhzYCBfMg+u+BJdcB2l+rr+crv5oa2LMY9dR1r9zhIKcCFdfOqlv7CGdWw/nqr2rh1d2HeHlXYfZ39ROQ0sHDS0dNB7v4PCJTgY6jBTnZVMZBMdAodL7mFiYO6rjOV09cY62dnLkxMlHbwAcGeBxtLWTrp6xcZzMMvrCJJIUMjVVpXznjkGP9wNSWKRCPA6bfpzonjq6G6ZdlQiNqZel9nNFxrDunjhHTnRyqKWDhuMdfUHSGyYNSctb2rtP298MyifknhImZ2q5lE7IOeWva3fnRGcPR453cqS1kyMnOjh8PHGA7/3rv//Bv3mAGnqV5GczsSiPsgk5lBcmgqysMPeMPwtzI/TEne7eR0+crh6nOx6nu+fUZT1xp6t3eU882CfY/pR9krbv3a7v58n36Fvf48TKCvjMtTPO6b+fwiKVujth/Xfhha/AiQaY9cHENRqVM0fn80XSVHtXzynh0fc4fmrAHGrpoLP79As/cyJGRVEeJfk5NLV1ceRE5xkvEM2JGOWFuZQX5lFemDj4lwch0Pu6rDCHicHPsgm5p10tnwkUFqOh4zj85zfhN1+HrlZYcBtc8+cQrRrdOkTGmd5xglMCJSlUmtu6KJ2Qc/Kv/Am5TCwKfgYH/6K87LSfEmY0KCxG04lGeOGrsPY7kBWBJXfDVZ+HCeXh1CMiMkRDDYvMa3OlQmEF3PggfGY9zPl9+M034OsL4KW/h87WsKsTETlvCouRVDYNPvwt+PRLMPW9ibOnvrEoMb7Rc+ZBNRGRsU5hkQoXzoPbnoA7n4FoDP71c/D/vRc2r2bAcwxFRMY4hUUqVV8Jn/g5fOxfwLLgiY/Dd66DXS+GXZmIyLAoLFLNDGZ/EO75Ddz0D9C8D/75g/CDj8D+DWFXJyIyJAqL0RLJhkUfh8++Cr/7ANSvg28thac+CUd2hV2diMhZpTQszOwGM9tqZjvM7L4B1ueZ2f8J1r9sZtVJ6/48WL7VzH4vlXWOqpwCuPJz8Lk3EqfXbvk3+IfL4Jk/g+MNYVcnIjKglF1nYWYRYBvwu0A9sBZY6e6bk7b5L0Ctu3/azG4FbnH3j5nZHOBxYAkwBfgFMNPde870eWl7D+7kKdEtCwpKE1Oj9z5yep/nnWV5wcn1Ofmnbpedlwio/vv3Lo/kpn5+K3fw+OmPeE/S6wG2wRPfiWWBRYKflrRsgEdWRPN1iQzDUK+zSOXtsZYAO9x9Z1DQKuBmYHPSNjcDXwqePwn8gyUuubwZWOXuHcAuM9sRvN9/prDecJRMhg99Da64F177PrQ3Q3d78OhI3MmvuwNajwy8vLstOLCeh/4hlJUD9D94e7+De/wsB/p+24XhbIFilhQ+Z1qfFD4ofOQsxsIfJxfMhRWPpvQjUhkWVcCepNf1wOVn2sbdu82sCZgYLP9tv31Pm0PDzO4G7ga46KKLRqzwUFTMSIxlnIuergFCpP3UR1dS0HQnbTPQ8p7OkTnInm39aS2GAdYDkBxSZ2ihDPnhA7RoBgm9+Bkbs6PMUWiNRWPkVPjSaSn/iLS+8a67PwI8AoluqJDLCU8kJ/HQ3fxEJEVSOcC9F5ia9DoWLBtwGzPLBqLA4SHuKyIioySVYbEWmGFm080sF7gVWN1vm9XAHcHzFcCvPDHivhq4NThbajowA3glhbWKiMhZpKwbKhiDuBd4FogAj7r7JjN7AFjn7quBfwK+HwxgHyERKATbPUFiMLwb+K9nOxNKRERSS1OUi4hkME1RLiIiI0ZhISIig1JYiIjIoBQWIiIyqHEzwG1mDcA75/EWFUDjCJWT7vRdnErfx6n0fZw0Hr6Lae5eOdhG4yYszpeZrRvKGQGZQN/FqfR9nErfx0mZ9F2oG0pERAalsBARkUEpLE56JOwCxhB9F6fS93EqfR8nZcx3oTELEREZlFoWIiIyKIWFiIgMKuPDwsxuMLOtZrbDzO4Lu54wmdlUM1tjZpvNbJOZfS7smsJmZhEze83M/i3sWsJmZqVm9qSZvWVmW8zsirBrCpOZfT74d/KmmT1uZvlh15RKGR0WZhYBvgncCMwBVprZnHCrClU38P+4+xzgvcB/zfDvA+BzwJawixgjvgb8X3efBcwng78XM6sCPgvUufs8ErdhuDXcqlIro8MCWALscPed7t4JrAJuDrmm0Lj7fnd/NXjeQuJgcNq9zzOFmcWADwDfCbuWsJlZFFhG4h40uHunux8Lt6rQZQMFwV0+JwD7Qq4npTI9LKqAPUmv68ngg2MyM6sGFgIvh1tJqB4G/hsQD7uQMWA60AA8FnTLfcfMCsMuKizuvhf4KvAusB9ocvf/CLeq1Mr0sJABmFkR8BTwJ+7eHHY9YTCzDwKH3H192LWMEdnAIuAf3X0hcALI2DE+Mysj0QsxHZgCFJrZ7eFWlVqZHhZ7galJr2PBsoxlZjkkguJf3P3HYdcToiuBm8xsN4nuyd8xsx+EW1Ko6oF6d+9taT5JIjwy1XXALndvcPcu4MfA+0KuKaUyPSzWAjPMbLqZ5ZIYoFodck2hMTMj0Se9xd0fCrueMLn7n7t7zN2rSfx/8St3H9d/OZ6Nux8A9pjZpcGia4HNIZYUtneB95rZhODfzbWM8wH/7LALCJO7d5vZvcCzJM5meNTdN4VcVpiuBD4ObDSz14Nlf+Huz4RYk4wdnwH+JfjDaidwV8j1hMbdXzazJ4FXSZxF+BrjfOoPTfchIiKDyvRuKBERGQKFhYiIDEphISIig1JYiIjIoBQWIiIyKIWFyDCYWY+ZvZ70GLGrmM2s2szeHKn3ExlJGX2dhcg5aHP3BWEXITLa1LIQGQFmttvMvmJmG83sFTO7JFhebWa/MrMNZvZLM7soWH6BmT1tZm8Ej96pIiJm9u3gPgn/YWYFof1SIkkUFiLDU9CvG+pjSeua3L0G+AcSM9YCfAP4Z3evBf4F+Hqw/OvA8+4+n8QcS70zB8wAvunuc4FjwEdS/PuIDImu4BYZBjM77u5FAyzfDfyOu+8MJmM84O4TzawRmOzuXcHy/e5eYWYNQMzdO5Leoxr4ubvPCF7/dyDH3f9n6n8zkbNTy0Jk5PgZng9HR9LzHjSuKGOEwkJk5Hws6ed/Bs9/w8nbbd4GvBg8/yVwD/Td5zs6WkWKnAv91SIyPAVJM/JC4p7UvafPlpnZBhKtg5XBss+QuLvcn5G401zvTK2fAx4xs0+QaEHcQ+KOayJjksYsREZAMGZR5+6NYdcikgrqhhIRkUGpZSEiIoNSy0JERAalsBARkUEpLEREZFAKCxERGZTCQkREBvX/A8Bh8DbF6hUGAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluation = model.evaluate(x = X_test, y = y_test, verbose=1) \n",
    "loss = evaluation[0]\n",
    "acc = evaluation[1]\n",
    "print('LOSS ON TEST DATA='+str(loss))\n",
    "print('ACCURACY ON TEST DATA='+str(acc))\n",
    "\n",
    "\n",
    "# Plot training & validation loss values\n",
    "plt.plot(history.history['loss'], label='Train')\n",
    "plt.plot(history.history['val_loss'], label='Validation')\n",
    "plt.title('Model loss')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Validation'], loc='upper left')\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.99      1.00       485\n",
      "           1       0.97      0.99      0.98        94\n",
      "\n",
      "   micro avg       0.99      0.99      0.99       579\n",
      "   macro avg       0.98      0.99      0.99       579\n",
      "weighted avg       0.99      0.99      0.99       579\n",
      "\n",
      "Accuracy: 0.9930915371329879\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAF4BJREFUeJzt3X20XXV95/H3xwREBYMasJoEggJV6hOaAo5rKTMiRUaDy/pARrQ4lLRWWh2trbPqKKVVq452bMUqFYqiguBYV1pj8QnNaEUTB6SAQiPykIADCAQRDE/f+WPvS46XZN+T6933nJu8X2vdxX74nb2/+8fN+dy9f+fsnapCkqRtecioC5AkjTeDQpLUyaCQJHUyKCRJnQwKSVIng0KS1Mmg0JyS5LIkh0/RZp8kdySZN0tl9S7J1UmOaKdPTvLJUdeknYdBoRnRvpHd1b5B/78kZybZfab3U1W/UVVfn6LNtVW1e1XdN9P7b9+k72mP87Yk/5rk2TO9H2mcGBSaSS+uqt2BZwLLgLdNbpDGXP+9+0x7nAuBC4DzRlzPjEsyf9Q1aHzM9X+wGkNVtRH4IvAUgCRfT/LOJN8C7gSekGRBktOT3JBkY5K/HLxUlOTEJD9I8rMklyd5Zrt88BLMIUnWJbm9PYv5QLt8aZKaeLNL8vgkq5LckmR9khMH9nNyknOTfKLd12VJlg15nPcCnwIWJdlrYJsvSnLxwBnH0wbWLUnyuSQ3Jflpkg+1y5+Y5GvtspuTfCrJntPp/yTHtPu/PcmPkhw1ue8Gjv2Tk/rshCTXAl9L8sUkJ03a9veTvLSdflKSL7f9ekWSV0ynXo0/g0IzLskS4GjgooHFrwZWAnsA1wBnAvcC+wMHA0cCv9u+/uXAycBrgEcCy4GfbmVXHwQ+WFWPBJ4InLuNks4BNgCPB14GvCvJfxpYv7xtsyewCvjQkMe5a1vjT4Fb22UHA2cAvwc8BvgosCrJQ9sg/Of2+JcCi9r9AgR4d1vjk4ElbR9slySHAJ8A3tIez3OBq7djE89r9/9bwNnAioFtHwTsC3whySOALwOfBvYGjgU+3LbRDsag0Ez6fJLbgG8C3wDeNbDuzKq6rP0r/NE0QfLGqvp5Vd0I/DXNmw00gfHeqlpbjfVVdc1W9ncPsH+ShVV1R1VdOLlBG1rPAf60qn5RVRcDH6N5g5/wzapa3Y5pnAU8fYrjfEV7nHcBJwIva48LmjD8aFV9p6ruq6qPA5uBw4BDaILgLe1x/6KqvgnQHuOXq2pzVd0EfIDmTXt7nQCc0W7r/qraWFU/3I7Xn9zWdhfwj8AzkuzbrnsV8Lmq2gy8CLi6qv6hqu6tqouA/w28fBo1a8wZFJpJL6mqPatq36r6g/bNZsJ1A9P7ArsAN7SXZ26j+ct773b9EuBHQ+zvBOBA4IdJ1iZ50VbaPB64pap+NrDsGpq/5if8ZGD6TmC3JPOTvKodtL4jyRcH2pxbVXsCjwUuBZ416djePHFc7bEtaetYAlwzECoPSPLYJOe0l+FuBz5JMwayvYbtu2154P9T22dfYEuAr6C51AbNcR466ThfBfzar7BvjSkHrDRbBm9TfB3NX9kLt/am2a5/4pQbrPp3YEU7OP5S4LNJHjOp2fXAo5PsMRAW+wAbh9j+p9jyxri19TcnWQmsS/Lpqrqhrf2dVfXOye3bT0ftk2T+Vo77XTR99NSquiXJSxjyEtgkXX33c+DhA/Nbe1OffDvps4F3JFkD7EYzeD+xn29U1QumUaPmGM8oNOvaN9QvAe9P8sgkD2kHcycutXwM+OMkz2o+JJX9By5/PCDJcUn2qqr7gdvaxfdP2td1wL8C706yWzuwfALNX+wzcSxXAOcDf9Iu+nvg95Mc2tb+iCT/OckewHeBG4C/apfvluQ57ev2AO4ANiVZRDPGMB2nA69N8vy2XxcleVK77mLg2CS7tAP2Lxtie6tpzh5Oofm010T//jNwYJJXt9vbJclvJnnyNOvWGDMoNCqvAXYFLqcZCP4s8DiAqjoPeCfNQOnPgM/TjGtMdhRwWZI7aAa2j510uWvCCprB4+tprru/o6q+MoPH8j5gZZK9q2odzbjFh9rjWg8cD9COgbyYZgD/WpoB9le22/hzmo8Vb6K53PO56RRSVd8FXksz5rOJZqxoImT/B83Zxq3t/j49xPY2t7UcMdi+PTs7kuay1PU0l+/eAzx0OnVrvMUHF0mSunhGIUnqZFBIkjoZFJKkTgaFJKnTnPsexcKFC2vp0qWjLkOS5pTvfe97N1fVXlO3fLA5FxRLly5l3bp1oy5DkuaUJFu7Dc5QvPQkSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjr1FhRJzkhyY5JLt7E+Sf4mzTOML0n7TGRJ0njp84ziTJrbQG/LC4ED2p+VwN/1WIskaZp6C4qqWgPc0tHkGOAT7TORLwT2TPK4qbZ7990zVaEkaRijHKNYxC8/R3kDv/wc4wckWZlkXZJ1N9xw66wUJ0lqzInB7Ko6raqWVdWyBQseNepyJGmnMsqg2AgsGZhfzBAPvJckza5RBsUq4DXtp58OAzZV1Q0jrEeStBW93T02ydnA4cDCJBuAdwC7AFTVR4DVwNE0D5+/k+aB8JKkMdNbUFTViinWF/D67d3u/ffDlVdOuyxJ2ik8+tGwcOHMbGvOPY8CYM2aUVcgSeNr8+YmKFZ0/rk+vDkXFPPnw8EHj7oKSRpf11wDt3R9i207zYmPx0qSRsegkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdeo1KJIcleSKJOuTvHUr6/dJckGSi5JckuToPuuRJG2/3oIiyTzgVOCFwEHAiiQHTWr2NuDcqjoYOBb4cF/1SJKmp88zikOA9VV1VVXdDZwDHDOpTQGPbKcXANf3WI8kaRrm97jtRcB1A/MbgEMntTkZ+FKSPwQeARyxtQ0lWQmsBNh7731mvFBJ0raNejB7BXBmVS0GjgbOSvKgmqrqtKpaVlXLFizYa9aLlKSdWZ9BsRFYMjC/uF026ATgXICq+jawG7Cwx5okSdupz6BYCxyQZL8ku9IMVq+a1OZa4PkASZ5MExQ39ViTJGk79RYUVXUvcBJwPvADmk83XZbklCTL22ZvBk5M8n3gbOD4qqq+apIkbb8+B7OpqtXA6knL3j4wfTnwnD5rkCT9akY9mC1JGnMGhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqNH/YhkkWAfsOvqaq1vRRlCRpfAwVFEneA7wSuBy4r11cQGdQJDkK+CAwD/hYVf3VVtq8Aji53d73q+q/DFu8JKl/w55RvAT49araPOyGk8wDTgVeAGwA1iZZVVWXD7Q5APjvwHOq6tYkew9fuiRpNgw7RnEVsMt2bvsQYH1VXVVVdwPnAMdManMicGpV3QpQVTdu5z4kST0b9oziTuDiJF8FHjirqKo/6njNIuC6gfkNwKGT2hwIkORbNJenTq6qfxmyJknSLBg2KFa1P33s/wDgcGAxsCbJU6vqtsFGSVYCKwH23nufHsqQJG3LUEFRVR9PsivtGQBwRVXdM8XLNgJLBuYXt8sGbQC+027rx0mupAmOtZP2fxpwGsCBBy6rYWqWJM2MocYokhwO/DvN4PSHgSuTPHeKl60FDkiyXxsyx/Lgs5LP05xNkGQhTRBdNWzxkqT+DXvp6f3AkVV1BUCSA4GzgWdt6wVVdW+Sk4DzacYfzqiqy5KcAqyrqlXtuiOTTHzs9i1V9dPpH44kaaYNGxS7TIQEQFVdmWTKT0FV1Wpg9aRlbx+YLuBN7Y8kaQwNGxTrknwM+GQ7/ypgXT8lSZLGybBB8Trg9cDEx2H/D81YhSRpBzfsp542Ax9ofyRJO5HOoEhyblW9Ism/0dyL6ZdU1dN6q0ySNBamOqN4Q/vfF/VdiCRpPHV+j6Kqbmgnbwauq6prgIcCTweu77k2SdIYGPamgGuA3dpnUnwJeDVwZl9FSZLGx7BBkaq6E3gp8OGqejnwG/2VJUkaF0MHRZJn03x/4gvtsnn9lCRJGifDBsUbaR4w9I/tbTieAFzQX1mSpHEx7PcovgF8Y2D+KrZ8+U6StAOb6nsU/6uq3pjkn9j69yiW91aZJGksTHVGcVb73//ZdyGSpPHUGRRV9b12ch1wV1XdD5BkHs33KSRJO7hhB7O/Cjx8YP5hwFdmvhxJ0rgZNih2q6o7Jmba6Yd3tJck7SCGDYqfJ3nmxEySZwF39VOSJGmcDPs8ijcC5yW5Hgjwa8Are6tKkjQ2hv0exdokTwJ+vV10RVXd019ZkqRxMdSlpyQPB/4UeENVXQosTeKtxyVpJzDsGMU/AHcDz27nNwJ/2UtFkqSxMmxQPLGq3gvcA9DeSTa9VSVJGhvDBsXdSR5GexuPJE8ENvdWlSRpbAz7qad3AP8CLEnyKeA5wPF9FSVJGh9TBkWSAD+keWjRYTSXnN5QVTf3XJskaQxMGRRVVUlWV9VT2fLQIknSTmLYMYr/m+Q3e61EkjSWhh2jOBQ4LsnVwM9pLj9VVT2tr8IkSeNh2KD4rV6rkCSNramecLcb8PvA/sC/AadX1b2zUZgkaTxMNUbxcWAZTUi8EHh/7xVJksbKVJeeDmo/7USS04Hv9l+SJGmcTHVG8cAdYr3kJEk7p6mC4ulJbm9/fgY8bWI6ye1TbTzJUUmuSLI+yVs72v12kkqybHsPQJLUr85LT1U1b7obTjIPOBV4AbABWJtkVVVdPqndHsAbgO9Md1+SpP4M+4W76TgEWF9VV1XV3cA5wDFbafcXwHuAX/RYiyRpmvoMikXAdQPzG9plD2ifw72kqjpvDZJkZZJ1SdZt2nTTzFcqSdqmPoOiU5KHAB8A3jxV26o6raqWVdWyBQv26r84SdID+gyKjcCSgfnF7bIJewBPAb7e3hrkMGCVA9qSNF76DIq1wAFJ9kuyK3AssGpiZVVtqqqFVbW0qpYCFwLLq2pdjzVJkrZTb0HRfu/iJOB84AfAuVV1WZJTkizva7+SpJk17E0Bp6WqVgOrJy17+zbaHt5nLZKk6RnZYLYkaW4wKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktSp16BIclSSK5KsT/LWrax/U5LLk1yS5KtJ9u2zHknS9ustKJLMA04FXggcBKxIctCkZhcBy6rqacBngff2VY8kaXr6PKM4BFhfVVdV1d3AOcAxgw2q6oKqurOdvRBY3GM9kqRp6DMoFgHXDcxvaJdtywnAF7e2IsnKJOuSrNu06aYZLFGSNJWxGMxOchywDHjf1tZX1WlVtayqli1YsNfsFidJO7n5PW57I7BkYH5xu+yXJDkC+DPgeVW1ucd6JEnT0OcZxVrggCT7JdkVOBZYNdggycHAR4HlVXVjj7VIkqapt6CoqnuBk4DzgR8A51bVZUlOSbK8bfY+YHfgvCQXJ1m1jc1Jkkakz0tPVNVqYPWkZW8fmD6iz/1Lkn51YzGYLUkaX72eUUiSRmPzZrjyypnZlkEhSTuYBQvgJz+BNWsGl+7xiOluz6CQpB3MnnvCoYdOXjpv3nS35xiFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpU69BkeSoJFckWZ/krVtZ/9Akn2nXfyfJ0j7rkSRtv96CIsk84FTghcBBwIokB01qdgJwa1XtD/w18J6+6pEkTU+fZxSHAOur6qqquhs4BzhmUptjgI+3058Fnp8kPdYkSdpO83vc9iLguoH5DcCh22pTVfcm2QQ8Brh5sFGSlcDKdu6eZcsedXUvFc85mxfAQzeNuorxYF9sYV9sYV9scfvi6b6yz6CYMVV1GnAaQJJ1VbcuG3FJY6HpizvtC+yLQfbFFvbFFknWTfe1fV562ggsGZhf3C7bapsk84EFwE97rEmStJ36DIq1wAFJ9kuyK3AssGpSm1XA77TTLwO+VlXVY02SpO3U26WndszhJOB8YB5wRlVdluQUYF1VrQJOB85Ksh64hSZMpnJaXzXPQfbFFvbFFvbFFvbFFtPui/gHvCSpi9/MliR1MigkSZ3GNii8/ccWQ/TFm5JcnuSSJF9Nsu8o6pwNU/XFQLvfTlJJdtiPRg7TF0le0f5uXJbk07Nd42wZ4t/IPkkuSHJR++/k6FHU2bckZyS5Mcml21ifJH/T9tMlSZ451Iaraux+aAa/fwQ8AdgV+D5w0KQ2fwB8pJ0+FvjMqOseYV/8R+Dh7fTrdua+aNvtAawBLgSWjbruEf5eHABcBDyqnd971HWPsC9OA17XTh8EXD3qunvqi+cCzwQu3cb6o4EvAgEOA74zzHbH9YzC239sMWVfVNUFVXVnO3shzXdWdkTD/F4A/AXNfcN+MZvFzbJh+uJE4NSquhWgqm6c5RpnyzB9UcAj2+kFwPWzWN+sqao1NJ8g3ZZjgE9U40JgzySPm2q74xoUW7v9x6Jttamqe4GJ23/saIbpi0En0PzFsCOasi/aU+klVfWF2SxsBIb5vTgQODDJt5JcmOSoWatudg3TFycDxyXZAKwG/nB2Shs72/t+AsyRW3hoOEmOA5YBzxt1LaOQ5CHAB4DjR1zKuJhPc/npcJqzzDVJnlpVt420qtFYAZxZVe9P8mya7289paruH3Vhc8G4nlF4+48thukLkhwB/BmwvKo2z1Jts22qvtgDeArw9SRX01yDXbWDDmgP83uxAVhVVfdU1Y+BK2mCY0czTF+cAJwLUFXfBnYDFs5KdeNlqPeTycY1KLz9xxZT9kWSg4GP0oTEjnodGqboi6raVFULq2ppVS2lGa9ZXlXTvhnaGBvm38jnac4mSLKQ5lLUVbNZ5CwZpi+uBZ4PkOTJNEFx06xWOR5WAa9pP/10GLCpqm6Y6kVjeemp+rv9x5wzZF+8D9gdOK8dz7+2qpaPrOieDNkXO4Uh++J84MgklwP3AW+pqh3urHvIvngz8PdJ/hvNwPbxO+IflknOpvnjYGE7HvMOYBeAqvoIzfjM0cB64E7gtUNtdwfsK0nSDBrXS0+SpDFhUEiSOhkUkqROBoUkqZNBIUnqZFBIkyS5L8nFSS5N8k9J9pzh7R+f5EPt9MlJ/ngmty/NNINCerC7quoZVfUUmu/ovH7UBUmjZFBI3b7NwE3Tkrwlydr2Xv5/PrD8Ne2y7yc5q1324vZZKRcl+UqSx46gfulXNpbfzJbGQZJ5NLd9OL2dP5LmXkmH0NzPf1WS59LcY+xtwH+oqpuTPLrdxDeBw6qqkvwu8Cc03xCW5hSDQnqwhyW5mOZM4gfAl9vlR7Y/F7Xzu9MEx9OB86rqZoCqmngewGLgM+39/ncFfjw75Uszy0tP0oPdVVXPAPalOXOYGKMI8O52/OIZVbV/VZ3esZ2/BT5UVU8Ffo/mRnTSnGNQSNvQPjXwj4A3t7eyPx/4r0l2B0iyKMnewNeAlyd5TLt84tLTArbcwvl3kOYoLz1JHarqoiSXACuq6qz2FtXfbu/SewdwXHun0ncC30hyH82lqeNpnqp2XpJbacJkv1Ecg/Sr8u6xkqROXnqSJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSp/8PQL0DXqG31WQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluate(model, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 100 features only"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_up(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/spiros/env/lib/python3.5/site-packages/ipykernel_launcher.py:6: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(200, kernel_initializer=\"glorot_normal\", activation=\"relu\", input_shape=(100,))`\n",
      "  \n",
      "/home/spiros/env/lib/python3.5/site-packages/ipykernel_launcher.py:8: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(100, kernel_initializer=\"glorot_normal\", activation=\"relu\")`\n",
      "  \n",
      "/home/spiros/env/lib/python3.5/site-packages/ipykernel_launcher.py:10: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(50, kernel_initializer=\"glorot_normal\", activation=\"relu\")`\n",
      "  # Remove the CWD from sys.path while we load stuff.\n",
      "/home/spiros/env/lib/python3.5/site-packages/ipykernel_launcher.py:12: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(10, kernel_initializer=\"glorot_normal\", activation=\"relu\")`\n",
      "  if sys.path[0] == '':\n",
      "/home/spiros/env/lib/python3.5/site-packages/ipykernel_launcher.py:14: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1, kernel_initializer=\"glorot_normal\", activation=\"sigmoid\")`\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 200)               20200     \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 200)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 100)               20100     \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 50)                5050      \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 10)                510       \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 10)                0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 45,871\n",
      "Trainable params: 45,871\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 2082 samples, validate on 232 samples\n",
      "Epoch 1/15\n",
      "2082/2082 [==============================] - 1s 335us/step - loss: 0.4360 - acc: 0.8261 - val_loss: 0.1808 - val_acc: 0.8233\n",
      "Epoch 2/15\n",
      "2082/2082 [==============================] - 0s 190us/step - loss: 0.1838 - acc: 0.8588 - val_loss: 0.1354 - val_acc: 0.9784\n",
      "Epoch 3/15\n",
      "2082/2082 [==============================] - 0s 165us/step - loss: 0.1237 - acc: 0.9568 - val_loss: 0.0642 - val_acc: 0.9784\n",
      "Epoch 4/15\n",
      "2082/2082 [==============================] - 0s 180us/step - loss: 0.0977 - acc: 0.9741 - val_loss: 0.0461 - val_acc: 0.9871\n",
      "Epoch 5/15\n",
      "2082/2082 [==============================] - 0s 171us/step - loss: 0.0826 - acc: 0.9765 - val_loss: 0.0540 - val_acc: 0.9828\n",
      "Epoch 6/15\n",
      "2082/2082 [==============================] - 0s 186us/step - loss: 0.0635 - acc: 0.9793 - val_loss: 0.0542 - val_acc: 0.9828\n",
      "Epoch 7/15\n",
      "2082/2082 [==============================] - 0s 178us/step - loss: 0.0528 - acc: 0.9822 - val_loss: 0.0797 - val_acc: 0.9741\n",
      "Epoch 8/15\n",
      "2082/2082 [==============================] - 0s 179us/step - loss: 0.0489 - acc: 0.9861 - val_loss: 0.0787 - val_acc: 0.9784\n",
      "Epoch 9/15\n",
      "2082/2082 [==============================] - 0s 172us/step - loss: 0.0573 - acc: 0.9837 - val_loss: 0.0687 - val_acc: 0.9828\n",
      "Epoch 10/15\n",
      "2082/2082 [==============================] - 0s 168us/step - loss: 0.0480 - acc: 0.9813 - val_loss: 0.0736 - val_acc: 0.9741\n",
      "Epoch 11/15\n",
      "2082/2082 [==============================] - 0s 168us/step - loss: 0.0379 - acc: 0.9875 - val_loss: 0.0735 - val_acc: 0.9784\n",
      "Epoch 12/15\n",
      "2082/2082 [==============================] - 0s 169us/step - loss: 0.0389 - acc: 0.9880 - val_loss: 0.0785 - val_acc: 0.9784\n",
      "Epoch 13/15\n",
      "2082/2082 [==============================] - 0s 168us/step - loss: 0.0387 - acc: 0.9846 - val_loss: 0.0849 - val_acc: 0.9784\n",
      "Epoch 14/15\n",
      "2082/2082 [==============================] - 0s 169us/step - loss: 0.0361 - acc: 0.9880 - val_loss: 0.0891 - val_acc: 0.9784\n",
      "Epoch 15/15\n",
      "2082/2082 [==============================] - 0s 169us/step - loss: 0.0452 - acc: 0.9866 - val_loss: 0.0816 - val_acc: 0.9828\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "\n",
    "input_shape = X_train.shape[1]\n",
    "model = Sequential()\n",
    "model.add(Dense(200, input_shape=(input_shape,), init='glorot_normal', activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(100, init='glorot_normal', activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(50, init='glorot_normal', activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(10, init='glorot_normal', activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(1, init='glorot_normal', activation='sigmoid'))\n",
    "model.summary()\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "history = model.fit(X_train, y_train, epochs=15, batch_size=16, verbose=1, validation_split=0.1)\n",
    "\n",
    "y_pred = model.predict(X_test)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "579/579 [==============================] - 0s 41us/step\n",
      "LOSS ON TEST DATA=0.10825832317551866\n",
      "ACCURACY ON TEST DATA=0.9740932642487047\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8XGXZ//HPNZO1ySRp2jRpmrQJpU2b7m0oBYRSQCgIrUAFKiib8sAjgiL6FPUHWkVxq0VFEbUgIlQEgbKJiIggS5uWEuie7umapU2aptmv3x/nJJ2G7JnJJJnr/XrNa2bOnOVKoPPNfd/n3EdUFWOMMaY9nlAXYIwxpu+zsDDGGNMhCwtjjDEdsrAwxhjTIQsLY4wxHbKwMMYY0yELC2N6QESyRERFJKIT614nIm/1dD/GhIKFhQkbIrJDRGpFZGiL5e+7X9RZoanMmL7PwsKEm+3AwqY3IjIJGBS6cozpHywsTLj5E/B5v/fXAo/6ryAiiSLyqIgUi8hOEfm2iHjcz7wi8lMRKRGRbcCnWtn2DyKyT0T2iMj3RcTb1SJFJF1EVohImYgUisgX/T6bKSL5IlIhIgdEZIm7PEZEHhORUhE5LCKrRCS1q8c2pjUWFibcvAskiMh490v8KuCxFuv8EkgETgJm44TL9e5nXwQuBqYBecCCFts+AtQDJ7vrnA98oRt1LgeKgHT3GD8QkXPcz+4H7lfVBGA08KS7/Fq37kxgCHAzcKwbxzbmYywsTDhqal18EtgA7Gn6wC9A7lLVI6q6A/gZ8Dl3lSuApaq6W1XLgB/6bZsKXAR8RVWPqupB4Ofu/jpNRDKBM4D/U9VqVV0L/J7jLaI64GQRGaqqlar6rt/yIcDJqtqgqqtVtaIrxzamLRYWJhz9CfgscB0tuqCAoUAksNNv2U5ghPs6Hdjd4rMmo9xt97ndQIeB3wLDulhfOlCmqkfaqOFGYCyw0e1qutjv53oFWC4ie0XkxyIS2cVjG9MqCwsTdlR1J85A90XA31p8XILzF/oov2UjOd762IfTzeP/WZPdQA0wVFWT3EeCqk7oYol7gWQR8bVWg6puUdWFOCH0I+ApEYlT1TpV/a6q5gKn43SXfR5jAsDCwoSrG4FzVPWo/0JVbcAZA7hXRHwiMgq4g+PjGk8Ct4lIhogMBhb5bbsP+AfwMxFJEBGPiIwWkdldKUxVdwNvAz90B60nu/U+BiAi14hIiqo2AofdzRpFZI6ITHK70ipwQq+xK8c2pi0WFiYsqepWVc1v4+MvA0eBbcBbwOPAMvez3+F09XwArOHjLZPPA1HAeuAQ8BQwvBslLgSycFoZzwD3qOo/3c/mAutEpBJnsPsqVT0GpLnHq8AZi3kDp2vKmB4Tu/mRMcaYjljLwhhjTIcsLIwxxnTIwsIYY0yHLCyMMcZ0KKjTIYvIXJyzNbzA71X1vjbWuxznLI5TVDXfnf1zA7DJXeVdVb25vWMNHTpUs7KyAlS5McaEh9WrV5eoakpH6wUtLNxzvR/AmVKhCFglIitUdX2L9XzA7cB7LXaxVVWndvZ4WVlZ5Oe3dSakMcaY1ojIzo7XCm431EygUFW3qWotzsRo81tZ73s4V6FWB7EWY4wxPRDMsBjBiXPoFHF8bhsARGQ6kKmqL7ayfbZ7U5o3ROTM1g4gIje5UzXnFxcXB6xwY4wxJwrZALd7f4AlwNda+XgfMFJVp+FMtfC4iCS0XElVH1LVPFXNS0npsMvNGGNMNwVzgHsPJ064loHfVNCAD5gI/FtEwJmqYIWIzHOnYagBUNXVIrIVZ5bNLg1K1NXVUVRURHW19XAFUkxMDBkZGURG2oSmxoSLYIbFKmCMiGTjhMRVONNCA6Cq5TjTQQMgIv8G7nTPhkrBmaK5QUROAsbgzNPTJUVFRfh8PrKysnADyfSQqlJaWkpRURHZ2dmhLscY00uC1g2lqvXArTiTrm0AnlTVdSKyWETmdbD5WUCBiKzFOaX2ZvdGM11SXV3NkCFDLCgCSEQYMmSItdaMCTNBvc5CVV8CXmqx7O421j3b7/XTwNOBqMGCIvDsd2pM+An7K7jrGxo5UFFNVW19qEsxxpg+K+zDQgQOVFRTWR34sCgtLWXq1KlMnTqVtLQ0RowY0fy+tra2U/u4/vrr2bRpU8crGmNMEAW1G6o/8Ho8RHk9VNcF/oZiQ4YMYe3atQB85zvfIT4+njvvvPOEdVQVVcXjaT23H3744YDXZYwxXRX2LQuAmEgv1fUNvXa8wsJCcnNzufrqq5kwYQL79u3jpptuIi8vjwkTJrB48eLmdT/xiU+wdu1a6uvrSUpKYtGiRUyZMoXTTjuNgwcP9lrNxpjwFjYti+8+v471eyta/ay2oZG6hkbiorr268hNT+CeSyZ0q56NGzfy6KOPkpeXB8B9991HcnIy9fX1zJkzhwULFpCbm3vCNuXl5cyePZv77ruPO+64g2XLlrFo0aLWdm+MMQFlLQvAIwIKjb14i9nRo0c3BwXAE088wfTp05k+fTobNmxg/fr1H9smNjaWCy+8EIAZM2awY8eO3irXGBPmwqZl0V4L4FhtA1sOHmFk8iCSBkX1Sj1xcXHNr7ds2cL999/PypUrSUpK4pprrmn1OoaoqOO1eb1e6uvtDC5jTO+wlgUQHelBEKrrem/cwl9FRQU+n4+EhAT27dvHK6+8EpI6jDGmLWHTsmiPR4ToiOCcEdUZ06dPJzc3l3HjxjFq1CjOOOOMkNRhjDFtEe3FfvpgysvL05Y3P9qwYQPjx4/v1Pa7So9SVdfAuLSPTW5rWtGV360xpu8SkdWqmtfRetYN5YqJ9FJb30hD48AIT2OMCSQLC1d0pBeAmhCNWxhjTF9mYeGKiXR+Fb15cZ4xxvQXFhauKK8Hj0jIBrmNMaYvs7BwiYgz7Yd1QxljzMdYWPiJCeHps8YY05cFNSxEZK6IbBKRQhFpcxIjEblcRFRE8vyW3eVut0lELghmnU2iI73UNzrzRAXKnDlzPnaR3dKlS7nlllva3CY+Ph6AvXv3smDBglbXOfvss2l5qnBLS5cupaqqqvn9RRddxOHDhztbujHGNAtaWIiIF3gAuBDIBRaKSG4r6/mA24H3/Jbl4tyzewIwF/i1u7+ginUHuQN5RtTChQtZvnz5CcuWL1/OwoULO9w2PT2dp556qtvHbhkWL730EklJSd3enzEmfAWzZTETKFTVbapaCywH5rey3veAHwH+kyHNB5arao2qbgcK3f0FVdPps8cC2BW1YMECXnzxxeabHe3YsYO9e/cybdo0zj33XKZPn86kSZN47rnnPrbtjh07mDhxolPTsWNcddVVjB8/nksvvZRjx441r3fLLbc0T29+zz33APCLX/yCvXv3MmfOHObMmQNAVlYWJSUlACxZsoSJEycyceJEli5d2ny88ePH88UvfpEJEyZw/vnnn3AcY0z4CuZ0HyOA3X7vi4BT/VcQkelApqq+KCJfb7Htuy22HdHyACJyE3ATwMiRI9uv5uVFsP/DdleJBEbX1hPhEYjoREMmbRJceF+7qyQnJzNz5kxefvll5s+fz/Lly7niiiuIjY3lmWeeISEhgZKSEmbNmsW8efPavL/1b37zGwYNGsSGDRsoKChg+vTpzZ/de++9JCcn09DQwLnnnktBQQG33XYbS5Ys4fXXX2fo0KEn7Gv16tU8/PDDvPfee6gqp556KrNnz2bw4MFs2bKFJ554gt/97ndcccUVPP3001xzzTUd/y6MMQNayAa4RcQDLAG+1t19qOpDqpqnqnkpKSkBqcsjQqAv4vbvimrqglJVvvnNbzJ58mTOO+889uzZw4EDB9rcx3/+85/mL+3JkyczefLk5s+efPJJpk+fzrRp01i3bl2r05v7e+utt7j00kuJi4sjPj6eyy67jDfffBOA7Oxspk6dCtg06MaY44LZstgDZPq9z3CXNfEBE4F/u39NpwErRGReJ7btug5aAE0OHT5G2dFaJqQntPlXflfNnz+fr371q6xZs4aqqipmzJjBI488QnFxMatXryYyMpKsrKxWpyXvyPbt2/npT3/KqlWrGDx4MNddd1239tMkOjq6+bXX67VuKGMMENyWxSpgjIhki0gUzoD1iqYPVbVcVYeqapaqZuF0O81T1Xx3vatEJFpEsoExwMog1tosJtJDoyq1ATwjKj4+njlz5nDDDTc0D2yXl5czbNgwIiMjef3119m5c2e7+zjrrLN4/PHHAfjoo48oKCgAnOnN4+LiSExM5MCBA7z88svN2/h8Po4cOfKxfZ155pk8++yzVFVVcfToUZ555hnOPPPMQP24xpgBKGgtC1WtF5FbgVcAL7BMVdeJyGIgX1VXtLPtOhF5ElgP1ANfUtVeuVouxh2rqK5rJLoz4xadtHDhQi699NLm7qirr76aSy65hEmTJpGXl8e4cePa3f6WW27h+uuvZ/z48YwfP54ZM2YAMGXKFKZNm8a4cePIzMw8YXrzm266iblz55Kens7rr7/evHz69Olcd911zJzpnDPwhS98gWnTplmXkzGmTTZFeQsNjcq6veWkJsSQmhATyBIHFJui3JiBwaYo7yavR4iK8Njss8YY48fCohUxEV6b9sMYY/wM+LDoTjdbTKSHmvpGGgdIF12gDZSuS2NM5w3osIiJiaG0tLTLX24xkV4UpcZaFx+jqpSWlhITY+M5xoSTYF5nEXIZGRkUFRVRXFzcpe3qGho5UFFDXWkkg6IG9K+oW2JiYsjIyAh1GcaYXjSgvwkjIyPJzs7u8nZ1DY1cfvffufETJ7HowvZPaTXGmHAwoLuhuivS62F0SjybD3z8gjZjjAlHFhZtyEnzsWm/hYUxxoCFRZvGpvrYc/gYFdV1oS7FGGNCzsKiDePSfABssa4oY4yxsGjL2FQnLDZaV5QxxlhYtCVjcCzx0RFstrAwxhgLi7aICGNT461lYYwxWFi0KyfNx6YDR2x6C2NM2LOwaEdOqo/DVXUUH6kJdSnGGBNSFhbtyElLAGyQ2xhjghoWIjJXRDaJSKGILGrl85tF5EMRWSsib4lIrrs8S0SOucvXisiDwayzLTnu6bN2cZ4xJtwFbW4oEfECDwCfBIqAVSKyQlXX+632uKo+6K4/D1gCzHU/26qqU4NVX2ckx0WR4otmk11rYYwJc8FsWcwEClV1m6rWAsuB+f4rqGqF39s4oM+NJI+zaT+MMSaoYTEC2O33vshddgIR+ZKIbAV+DNzm91G2iLwvIm+IyJmtHUBEbhKRfBHJ7+o05J01NtXH5gNHaGjsczlmjDG9JuQD3Kr6gKqOBv4P+La7eB8wUlWnAXcAj4tIQivbPqSqeaqal5KSEpT6ctJ81NQ3squsKij7N8aY/iCYYbEHyPR7n+Eua8ty4NMAqlqjqqXu69XAVmBskOpsV05q0yB3RQdrGmPMwBXMsFgFjBGRbBGJAq4CVvivICJj/N5+CtjiLk9xB8gRkZOAMcC2INbaprGpPkRg0/7KUBzeGGP6hKCdDaWq9SJyK/AK4AWWqeo6EVkM5KvqCuBWETkPqAMOAde6m58FLBaROqARuFlVy4JVa3tio7yMSh7EpgPWsjDGhK+g3lZVVV8CXmqx7G6/17e3sd3TwNPBrK0rxqb67MI8Y0xYC/kAd38wLs3HjpKjVNc1hLoUY4wJCQuLTshJS6BRofCgjVsYY8KThUUn5KTFAzbthzEmfFlYdELWkDiiIjxstmk/jDFhysKiEyK8Hk5OsRshGWPCl4VFJ+XYHFHGmDBmYdFJOWk+9ldUU15VF+pSjDGm11lYdFLztB82bmGMCUMWFp10/EZIdiW3MSb8WFh00vDEGHwxEdayMMaEJQuLThIRclJtkNsYE54sLLqg6YwoVbsRkjEmvFhYdMG4NB8V1fXsr6gOdSnGGNOrLCy6YKx7RpRdnGeMCTcWFl0wLs25s+tmCwtjTJgJaliIyFwR2SQihSKyqJXPbxaRD0VkrYi8JSK5fp/d5W63SUQuCGadnZU4KJK0hBgb5DbGhJ2ghYV7W9QHgAuBXGChfxi4HlfVSao6FfgxsMTdNhfnNqwTgLnAr5tusxpqY9PsRkjGmPATzJbFTKBQVbepai2wHJjvv4Kq+l/hFgc0nWY0H1iuqjWquh0odPcXcuPSfBQWV1Lf0BjqUowxptcEMyxGALv93he5y04gIl8Ska04LYvburjtTSKSLyL5xcXFASu8PWNTfdTWN7KjtKpXjmeMMX1ByAe4VfUBVR0N/B/w7S5u+5Cq5qlqXkpKSnAKbGFc87Qf1hVljAkfwQyLPUCm3/sMd1lblgOf7ua2vebkYfF4xCYUNMaEl2CGxSpgjIhki0gUzoD1Cv8VRGSM39tPAVvc1yuAq0QkWkSygTHAyiDW2mkxkV6yhsTZhILGmLASEawdq2q9iNwKvAJ4gWWquk5EFgP5qroCuFVEzgPqgEPAte6260TkSWA9UA98SVUbglVrV+XYGVHGmDATtLAAUNWXgJdaLLvb7/Xt7Wx7L3Bv8Krrvpw0H39ft59jtQ3ERvWJM3qNMSaoQj7A3R/lpPpQhS0HrXVhjAkPFhbdkGNnRBljwoyFRTeMGhJHdITHwsIYEzYsLLrB6xHGpMbb6bPGmLBhYdFNOakJ1rIwxoQNC4tuykmL5+CRGg4drQ11KcYYE3QWFt2U497bwq63MMaEAwuLbmqaI2qzjVsYY8KAhUU3DfNFkxgbaS0LY0xYsLDoJhEhJ81nc0QZY8KChUUPjEvzsflAJara8crGGNOPWVj0wNhUH5U19ew5fCzUpRhjTFBZWPSADXIbY8KFhUUPjEl1wsIGuY0xA52FRQ8kxkaSnhhjV3IbYwY8C4secs6IsrAwxgxsQQ0LEZkrIptEpFBEFrXy+R0isl5ECkTkNREZ5fdZg4isdR8rWm7bV4xN87G1uJK6hsZQl2KMMUETtLAQES/wAHAhkAssFJHcFqu9D+Sp6mTgKeDHfp8dU9Wp7mNesOrsqXFpPuoalO0lR0NdijHGBE2nwkJERotItPv6bBG5TUSSOthsJlCoqttUtRZYDsz3X0FVX1fVKvftu0BG18oPvZxUZ44o64oyxgxknW1ZPA00iMjJwENAJvB4B9uMAHb7vS9yl7XlRuBlv/cxIpIvIu+KyKdb20BEbnLXyS8uLu7whwiG0cPi8HrEwsIYM6BFdHK9RlWtF5FLgV+q6i9F5P1AFSEi1wB5wGy/xaNUdY+InAT8S0Q+VNWt/tup6kM44UVeXl5ILqOOjvCSPTTOTp81xgxonW1Z1InIQuBa4AV3WWQH2+zBaYE0yXCXnUBEzgO+BcxT1Zqm5aq6x33eBvwbmNbJWntdTprPLswzxgxonQ2L64HTgHtVdbuIZAN/6mCbVcAYEckWkSjgKuCEs5pEZBrwW5ygOOi3fLDfGMlQ4AxgfSdr7XU5qT52lVVxtKY+1KUYY0xQdKobSlXXA7eB80UO+FT1Rx1sUy8itwKvAF5gmaquE5HFQL6qrgB+AsQDfxURgF3umU/jgd+KSCNOoN3n1tAn5bjTfmw5WMnUzI7G/Y0xpv/pVFiIyL+Bee76q4GDIvJfVb2jve1U9SXgpRbL7vZ7fV4b270NTOpMbX1Bjjvtx6b9FRYWxpgBqbPdUImqWgFcBjyqqqcCrX7Rh6ORyYOIjfTaILcxZsDqbFhEiMhw4AqOD3Abl8cjjE2Nt0FuY8yA1dmwWIwz9rBVVVe5p7NuCV5Z/c/YVJsjyhgzcHUqLFT1r6o6WVVvcd9vU9XLg1ta/5KT5qOkspaSypqOVzbGmH6ms9N9ZIjIMyJy0H08LSL9bmqOYBqX5kz7sdlaF8aYAaiz3VAP41wjke4+nneXGdfYtHjAboRkjBmYOhsWKar6sKrWu49HgJQg1tXvpMRHkxwXZeMWxpgBqbNhUSoi14iI131cA5QGs7D+RkTISfWxyc6IMsYMQJ0NixtwTpvdD+wDFgDXBammfqtpjqjGxpDMaWiMMUHT2bOhdqrqPFVNUdVhqvppwM6GaiEnzUdVbQNFh46FuhRjjAmontwpr92pPsLR2KZpP6wryhgzwPQkLCRgVQwQTRMKbtpfEeJKjDEmsHoSFtYx30J8dAQZg2PZdKAy1KUYY0xAtTvrrIgcofVQECA2KBX1czmpPmtZGGMGnHbDQlV9vVXIQJGT5uONzcXU1jcSFdGThpsxxvQd9m0WYDlpPuoblW0l1hVljBk4ghoWIjJXRDaJSKGILGrl8ztEZL2IFIjIayIyyu+za0Vki/u4Nph1BtLxQW47I8oYM3AELSxExAs8AFwI5AILRSS3xWrvA3mqOhl4Cvixu20ycA9wKjATuMe9nWufd9LQeCI8YnNEGWMGlGC2LGYChe505rXAcmC+/wqq+rqqVrlv3wWaZrK9AHhVVctU9RDwKjA3iLUGTFSEh9Ep8Tb7rDFmQAlmWIwAdvu9L3KXteVG4OWubCsiN4lIvojkFxcX97DcwBmb5rOWhTFmQOkTA9zuxIR5wE+6sp2qPqSqeaqal5LSdybBHZfmY8/hYxyprgt1KcYYExDBDIs9QKbf+wx32QlE5DzgW8A8Va3pyrZ9VdO0H5vt4jxjzAARzLBYBYwRkWwRiQKuwrmBUjMRmQb8FicoDvp99ApwvogMdge2z3eX9Qvj7IwoY8wA0+5FeT2hqvUicivOl7wXWKaq60RkMZCvqitwup3igb+KCMAud3bbMhH5Hk7gACxW1bJg1RpoI5JiiYvystkmFDTGDBBBCwsAVX0JeKnFsrv9Xp/XzrbLgGXBqy54PB5hTKqPjTbthzFmgOgTA9wD0bg0H5v2H0HV5ls0xvR/FhZBkpPm41BVHbvKqjpe2Rhj+jgLC1V490GoCuyQyHnjUxkU5eUbTxXQYLdZNcb0cxYWpYXw6t3w2GVQXR6w3WYmD+J78yfy3vYyfvWvwoDt1xhjQsHCYugYuOJR2P8RPLYAagJ3BtPlMzK4bNoI7n9tMyu395uTuYwx5mMsLABy5sKCZbBnNTx+FdQGbpxh8acnMjJ5ELcvf59DR2sDtl9jjOlNFhZNcufBZQ/BrrfhL1dDXXVAdhsfHcEvF06npLKGbzxdYGdHGWP6JQsLf5MWwPwHYOu/4K/XQn1gWgKTMhJZdOF4Xl1/gEff2RmQfRpjTG+ysGhp6mfh4p/D5r/D0zdAQ31AdnvDGVmcM24Y9764gXV7AzeQbowxvcHCojV5N8Dc+2DD8/DM/0BjQ493KSL8ZMFkBsdF8uUn3qeqNjAhZIwxvcHCoi2zboHzvgsfPQUrvgyNjT3e5ZD4aH5+5VS2lxzlnufWBaBIY4zpHRYW7fnEV+Dsb8LaP8OLdzgX8PXQ6aOHcuuck/nr6iKeW9tvZl03xoS5oE4kOCDM/gbUV8NbSyAiBub+EJwZcrvt9nPH8M7WUr71zEdMyUgia2hcgIo1xpjgsJZFR0Tg3Lth1pfgvd/AP7/T4xZGhNfD/Qun4fUIty1/n9r6nndxGWNMMFlYdIYIXHAv5N0I/10Kb/yox7sckRTLjy6fTEFROT95ZWMAijTGmOAJaliIyFwR2SQihSKyqJXPzxKRNSJSLyILWnzWICJr3ceKltv2OhG46Kcw9Rr49w/hzSU93uXciWl8btYofvfmdl7fdLDjDYwxJkSCFhYi4gUeAC4EcoGFIpLbYrVdwHXA463s4piqTnUf84JVZ5d4PDDvFzDpM/Dad+GdX/d4l9/61HjGpfm488kPOFgRmKvGjTEm0ILZspgJFKrqNlWtBZYD8/1XUNUdqloA9J9Oe48XPv0gjJ8Hr9wFq37fo93FRHr51WenUVXbwFf+stamMzfG9EnBDIsRwG6/90Xuss6KEZF8EXlXRD4d2NJ6yBsBl/8Bxl4IL34N1vypR7s7eZiP78zL5e2tpTz4xtYAFWmMMYHTlwe4R6lqHvBZYKmIjG65gojc5AZKfnFxce9WFxEFV/wRRp/rXLRX8Nce7e6KvEwumZLOklc3s3qnTWdujOlbghkWe4BMv/cZ7rJOUdU97vM24N/AtFbWeUhV81Q1LyUlpWfVdkdENFz5GGR9wpkWZP1z3d6ViHDvpRNJT4rhtifWUl5VF8BCjTGmZ4IZFquAMSKSLSJRwFVAp85qEpHBIhLtvh4KnAGsD1qlPRE1CBYuh4w8eOoG2PRyt3eVEBPJLxdO50BFNYv+ZtOZG2P6jqCFharWA7cCrwAbgCdVdZ2ILBaReQAicoqIFAGfAX4rIk0TJo0H8kXkA+B14D5V7ZthARAdD1f/FdImw5Ofh8LXur2rqZlJfP2CHF7+aD9/fm9XAIs0xpjuk4Hy12teXp7m5+eHtohjh+CPl0DJFrj6Kcg+s1u7aWxUrntkFe9tK+W5W89gXFpCgAs1xhiHiKx2x4fb1ZcHuPuf2MHwuWdhcDY8fiXserdbu/F4hJ99Zgq+mEhuffx9jtX2fIp0Y4zpCQuLQIsbCp9/DhKGw2MLoGxbt3aT4ovm51dOofBgJYtfsOnMjTGhZWERDL5Up4WBwktf7/bEg2eOSeGWs0fzxMrdvFCwN7A1GmNMF1hYBEtSJpzzbSj8J6x/ttu7ueOTY5k2Mom7nv6Q3WVVASzQGGM6z8IimE75IgyfAi8vguqKbu0i0uvhF1c5l5h8+Yn3qWvoPzOjGGMGDguLYPJGwMU/h8oD8Pq93d5NZvIg7rt8Mmt3H+Zn/9gcwAKNMaZzLCyCbcQMOOULsPIh2Pt+t3fzqcnDWTgzkwff2Mp/Nvfy1CbGmLBnYdEbzv1/EJcCz38FGrt/GuzdF09gzLB4bn5sNQ++sdXusGeM6TUWFr0hJhEu+AHsWwur/tDt3cRGeXnkhpmcPnoI9728kblL/8Mb1sowxvQCC4veMvFyGH0OvLYYKvZ1ezcjkmL5/bWn8PB1p9CoyrXLVvLFR/PtTCljTFBZWPSWptuyNtTCK9/s8e7mjBvGK189i2/MzeGtLSWcu+QNlry62a72NsYEhYVFbxoyGs66E9b9zbn+ooeiI7z879kn8687Z3PBhDR+8doWzlvyBn//aJ/NWGuMCSgLi952xu0w5GTnDns3HGYWAAAYfElEQVR1xwKyy+GJsfxy4TSW3zQLX0wENz+2hs/9YSWFB48EZP/GGGNh0dsiouFTS+DQDnjzZwHd9ayThvDClz/Bdy7JpaDoMHOXvsm9L67nSLXdSMkY0zMWFqFw0myYfCW8tRSKA3uRXYTXw3VnZPP6nWezYEYGv39rO+f87A3+tqbIuqaMMd1mYREq59/r3GXvxTu6PdFge4bER3Pf5ZN59n/PID0pljue/IAFD77DR3vKA34sY8zAF9SwEJG5IrJJRApFZFErn58lImtEpF5EFrT47FoR2eI+rg1mnSERnwLnfRd2vAkFfwnaYaZkJvHMLafz4wWT2VFylEt+9RbfeuZDDh2tDdoxjTEDT9DulCciXmAz8EmgCOee3Av9b48qIllAAnAnsEJVn3KXJwP5QB6gwGpghqoeaut4feJOeV3V2AjLzoey7XDrKhiUHNTDlR+rY+k/N/PoOzvxxURw5/k5LJw5Eq9HgnpcY0zf1RfulDcTKFTVbapaCywH5vuvoKo7VLUAaDlvxQXAq6pa5gbEq8DcINYaGh6PM9HgsUPwz+8E/XCJsZHcc8kEXrrtTMal+fj2sx8x71dvkb+jLOjHNsb0b8EMixHAbr/3Re6ygG0rIjeJSL6I5BcX99NpL9ImwaxbYM0fYdd7vXLInDQfT3xxFr/67DTKjtay4MF3uOMva+0qcGNMm/r1ALeqPqSqeaqal5KSEupyuu/suyAhA174KjT0zmmuIsLFk9N57Wuz+dKc0bxQsI8zf/w6l/36vyx7azsHKqp7pQ5jTP8QzLDYA2T6vc9wlwV72/4nOh4u+jEcXAfv/qZXDz0oKoKvXzCO179+Nl+/IIdjdY0sfmE9s374Glf+9h0ee3cnpZU1vVqTMabvCeYAdwTOAPe5OF/0q4DPquq6VtZ9BHihxQD3amC6u8oanAHuNjvX++UAd0tPLIRt/4YvvQdJI0NWRuHBSl4o2MvzH+xla/FRvB7h9NFDuGRyOhdMSCNxUGTIajPGBFZnB7iDFhZuERcBSwEvsExV7xWRxUC+qq4QkVOAZ4DBQDWwX1UnuNveADTNuHevqj7c3rEGRFgc3gUPnAonnQ0Lnwh1NagqG/cf4fkP9vJCwT52lVUR6RXOGpPCJVPSOS83lfjoiFCXaUz/oAraCB5vqCs5QZ8Ii940IMIC4L/3w6t3w1WPw7hPhbqaZqpKQVE5LxQ4wbGvvJroCA/njBvGxZPTOWfcMGKj+tY/AmN6RUMdHC2GI/uh8qBzG2X/x5Gm1wedWaeHnAzDxsGwXBg2HlLGQ/JJzm2YQ8DCor9qqIPfzobqcqc7Kjo+1BV9TGOjsmbXIZ7/YC8vfrifksoaBkV5OW98KhdPHs7snBSiIyw4TD+m6vwbrDwIlX4hcEIguJ9Vlba+j9hkiE+F+GHOsy8VPJFQshkOrneur8L9/vVGwdAcJzz8gyRxpHOKfRBZWPRnu95zLtY7/ctw/vdDXU27GhqV97aV8nzBPl7+aB+Hq+rwxURwfm4al0wZzhknDyXS269PujMDlarz5V+2FUq3+j1vc77I61uZFdobffyLvzkI0k4MhPhU5zbKEdHtH7+2yg2ODVC8wXk+uAHK/a4aiIyDlBw3PMa5YZILvuHOPXICwMKiv1txG7z/GPzPG861GP1AXUMj/y0s4fkP9vGPdfs5UlPP4EGRfCYvk8+fNoqMwYNCXaIJN6pOF9EJYbAVSrc5oVB39Pi63igYnO3cdyb5JOcL2ecXBPGpzi2SA/Ql3abqCije5LQ+/IOk8sDxdaIT3eBwH2mTYdRp3TqchUV/V1UGv8pz/qe94R9Bb4oGWnVdA//ZXMyza/fwyroDqCoXTEjjhk9kkzdqMBLsf3DdcWA9FK10vjQ8keCNdF57o9p43cY6nsh+99+r36sqayUQCp0WQk3F8fU8EZA0yg2E0c5z0+vEjD43+HyCqjK39dEUIhvhwDqoPgwZp8AXundDNQuLgWDtE/DszXDxUsi7PtTVdNuew8d49J0dLF+5m/JjdUwakcj1Z2Rx8eR0oiL6wJfq0RL41/edq+i15cwz3eSJODF0krMh81TInOk8+9ICc5xw0NjgdBdV7IHyIvd5j9NdU7HHCYTqw8fXF49z6nny6BNDIfkkZ7l3AJ36req0OKrLne6qbrCwGAhU4Y+XwP4CuHW1M1NtP1ZVW8/f1uzh4f9uZ2vxUVJ80Xxu1ig+e+pIhsZ30L8bDA11sOr38O8fQk0lzPwinPo/zpdNQ71z5kpDrbNey9eNda0vb35dd/x1fbXzV+CeNdDgXuCYNOrE8Eid0Lf/qg0WVSesK4qcAPhYIBTBkX2gLe4tHxUPCSMgcQQMzjoxGAZnQURUKH6afsnCYqAo3gy/OR0mXg6X/TbU1QREY6PyZmEJy97azhubi4mK8DB/SjrXn5FNbnpC7xRR+Br8/S4o2QQnzYG59zkDiMFUX+sE/653Yfd7zqOpHzoqHjLyjgdIxilO/3hvUHW6OCr2QMVep4+/6Swd3O7C5m7DHryvqfh4IFTsdcLUnzfKDYKM44GQMAISM4+/7o2xgzBhYTGQ/Ov78J+fwOdXOHfZG0AKD1byyNvbeXr1Ho7VNTDrpGRuOCObc8enBmfq9NKt8Mq3YPPLzmDmBT+AnAtD88Wj6lyIuXsl7HYD5MA6tytMnIHLzJmQOct5Tj6p63WqOrMalxc5X8wV7nPTl3ZbX9jBIh5n4PiEIMg4MRDihloQ9CILi4Gk7hj8epbTD37L2x2fktcPlVfVsXzVLv749g72llczMnkQ156exRV5GfhiAtDHXHPECdx3fu38/s76ujPbb1/7XdYcgT2rnQDZ9S4UrTo+QDto6PGWx8hZMHwq1FUd77Lx//JvDoe9Hz8FVLyQkO4+/P5yb3rEpzhf6s3fDe5zp9/T+ueRsc5ppiG6+My0zsJioNnyT/jz5TDnWzD7G6GuJmjqGxr5x/oDLHtrO/k7DxEfHcGCGRlcd3oWWUPjur7Dxkb44Al47btOl8/Uq+Hcu/vPAHNjozPesfu94y2Qsm1try9e5y/3hPQTQ8D/dfyw8BwfMa2ysBiI/nodbHwJ/vcdZzBvgCsoOszD/93BCwV7qW9Uzh03jBvOyOa00UM6d+rt7lXw8jdg7xpnDODCH8GIGcEvPNgqi51TfPcVQEyC20Jwu3Lihtlf7qZLLCwGoop98IDbd/3J70L27LDo2z1QUc1j7+7kz+/touxoLePSfHzutFHMHpvS+oV+Ffvgn/c49zb3DXfudT7pM3btgzGtsLAYqD76G7z4NThW5kxIlncjTF0IsYNDXVnQVdc1sGLtXpb9dzsb9x8BYERSLDOzkzk1O5lTMweRteUR5M0l0FgPp98Kn7ijT86vZUxfYWExkNVVw/rnnGsEilZCRKxzau0pNwyMbpYONE2dvnJ7GSu3l/HethJmHHubb0U8xkhPMe/Hncn26YvInTCZscN8eIJxVpUxA4SFRbjYVwD5f4CCvzrz3KRPc1obEy+HqDCYi+nAevTvi5Dtb3A4/mT+PPgWHjuYzb5y51TQxNhITslKZtZJyczMTiZ3eAIR/Xxiw4ZGZVtxJVuLjzIyeRBjU+P7/c9kQsfCItxUl0PBk05ro3ijc9HS1Ksh7wYYOibU1QVeVRm8/gMnKKMT4Jxvw4zrwRuBqlJ06JjT6theysrtZeworQIgPjqCGaMGN3ddTcpI7NPTqVfXNbD5wBHW7a1g3d5y1u2tYMO+Cqrrjk9LEhvpZdKIRKaOTGJKRhJTMhMZkRTbN+ffMn1OnwgLEZkL3I9zp7zfq+p9LT6PBh4FZgClwJWqukNEsoANwCZ31XdV9eb2jhX2YdFEFXa+7YTGhuedaSmyZ8MpN0LORcGfF6eh3jm188BHzgVmxRuhvgZQ97x7925hza/9ztHXxhbrtbW+wqEdzhQdp9wIZ98Fg5LbLetARfUJ4bH5QCUA0REepo1M4tTsIZyancyEEYkkxESE5Iu2orqO9XsrmoNh/d4KthyspKHR+R35YiLIHZ7AhPREJqQnMHpYPDtKjrJ292E+KDrMur0V1NY7ITI0PpqpmYlMzUxiSmYSkzOSSIwdQHMimYAJeViIiBfnHtyfBIpw7sG9UFXX+63zv8BkVb1ZRK4CLlXVK92weEFVJ3b2eBYWrag8CGsehdWPOJOuxafBjOtgxrXO6ZY9dbTkeCgcWOe8Lt50/Gpg8Tqn+EbFAeKcuSWe469bXUbn1otJgjNuh9TcbpVedrS2ecxj5Y5S1u+twP1OJirCw9C4KIbERzM03nkeEh9Fivs8JC6aoe5ng+OiunW/joMV1Se0FtbtrWBXWVXz5ym+aCakJzAhPYGJ6YlMSE8kM7n91kJtfSMb91ewdvfh5se24uNTcJ+UEsfUjKTmFsj44Ql9YyJHE1J9ISxOA76jqhe47+8CUNUf+q3zirvOOyISAewHUoBRWFgETmMDbPmH09oofM350s25EE75gtPq6OiU0voaJwSaAuHgeue1//z6ccOcyfBSJ0DqROd56FiIjAnuzxYgFdV1rN5xiM0HjlB2tJaSylpKKmsoPVpDqfu6rqH1fyuDB0U6gRIXxVBftF/QNIVLFAcqak4IhpLKmubtRw0Z5AZDIrluQAzzBeb3Vn6sjg+Lylm7+xBrd5ezdvfh5mNHeT3kpicwNTOpuQWSNWSQdV+Fmb4QFguAuar6Bff954BTVfVWv3U+ctcpct9vBU4F4oF1OC2TCuDbqvpmK8e4CbgJYOTIkTN27twZlJ9lQCnbDqsfhjV/ck6/TR7tjGtM/axz+m3FHr+Wgvso2Xx81k9vtDPhXlMgpE6AYRP6/Yy4HVFVKqrrKa2soaSy1nk+WkvJkeOB0hQqJZU1VFTXf2wfXo8wZli8GwiJTExPYHx6AgmBmM6kCz/H3vJqPvBrfXxYVM6xOue/b2JsJFMyk5gxcjB5WYOZmplEXLRd5DeQ9fewOALEq2qpiMwAngUmqGpFy+M0sZZFFzWdfpv/B2cqiYgYZ56k6vLj6ySO9GstuI/k0XaFcCfU1DdQdvR4gCTHRTE21UdMZN8bTK9vaGTLwUo+cMc+1uw8zOaDR1B1Am78cB95o5KZPmoweaMGk54UG+qSjetoTT0f7Smnpr6Rs8Z27w+2zoZFMP/V7wEy/d5nuMtaW6fI7YZKBErVSbAaAFVd7YbIWMDSIFAiY2DKlc5j/4dOS6Oh9ng30rDxEJsU6ir7regIL8MTYxme2Pe/WCO8HsYPT2D88ASumjkScLqv3t91iNU7D5G/4xB/WbWbR97eAUB6YgwzspKZMTKJvKxkxqX57NTdXlBT38DGfUcoKDrMB0XlFBQdpvBgJY0K44cndDssOiuYLYsInG6kc3FCYRXwWVVd57fOl4BJfgPcl6nqFSKSApSpaoOInAS86a5X1tbxrGVhTPDUNTSycd8R8neWkb/zEKt3HGJ/hXMiQ1yUl6kjk5gxKpkZowYzbWRSr3atDUQNjUrhwUo+KDpMQdFhCorK2bCvonncbEhcFJMzEpnsnio9aUQSKb7uzaAc8m4ot4iLgKU4p84uU9V7RWQxkK+qK0QkBvgTMA0oA65S1W0icjmwGKgDGoF7VPX59o5lYWFM72ka+8jfUdbc+ti43zmjTARyUn3kZQ0mzw2QjMFdu+5DValvVOoblNqGRuobGqlrUOoaGt3H8deKc4slEXGfweMeSwQEcZ7d5U3r0LTc3dbTYt346AjioyOC3mpSVXaWVrnB4LQYPtpT0TyOFB8dwaQRiUzOTGRKRhKTMwJ7HU2fCIveZGFhTGgdqa5j7e7DrN7pdF+9v+swlTXOQP8wXzRZQ+Koa3S+4JtCoOl1ywBo68yzUBgU5SUhJhJfTIT7iGx+ToiJICHW77PoyBPWSYiNJD464oQbee0vrz6hxVBQVE75sTrAOW17QnpCcyhMzkjipKFxQZ2ypi+MWRhjwogvJpIzx6Rw5hin77yhUdm4v4I1Ow+Rv/MQ+8uriY+OINLrIcIjREZ4iGrjdaRHiPR6iIxwlkVFeIjweIj0nvg60utxmgYKijrXdqpzu6VGVfeaT21epuoux/mLHprWV/czZ3mjKpU1DRypruNIdb3fcz2HqmrZVVbFkeo6Kqrrmy+EbE98tBMmdQ3afOqy1yOMTfVx4cQ0JrvhkJPm69Z1O73BwsIYExRej7hXmyfyudOyQl1O0NTUNzQHScWxE8Olwi9kKqrrUMVpOWQmkjs8kdiovnd2XFssLIwxpgeiI7xEx3sZGt/HbtEbYH2zvWOMMaZPsbAwxhjTIQsLY4wxHbKwMMYY0yELC2OMMR2ysDDGGNMhCwtjjDEdsrAwxhjToQEzN5SIFAM9ufvRUKAkQOUEW3+qFfpXvf2pVuhf9fanWqF/1duTWkepaofzmw+YsOgpEcnvzGRafUF/qhX6V739qVboX/X2p1qhf9XbG7VaN5QxxpgOWVgYY4zpkIXFcQ+FuoAu6E+1Qv+qtz/VCv2r3v5UK/SveoNeq41ZGGOM6ZC1LIwxxnTIwsIYY0yHwj4sRGSuiGwSkUIRWRTqetojIpki8rqIrBeRdSJye6hr6oiIeEXkfRF5IdS1dEREkkTkKRHZKCIbROS0UNfUFhH5qvv/wEci8oSIxIS6Jn8iskxEDorIR37LkkXkVRHZ4j4PDmWNTdqo9Sfu/wcFIvKMiCSFskZ/rdXr99nXRERFZGigjxvWYSEiXuAB4EIgF1goIrmhrapd9cDXVDUXmAV8qY/XC3A7sCHURXTS/cDfVXUcMIU+WreIjABuA/JUdSLgBa4KbVUf8wgwt8WyRcBrqjoGeM193xc8wsdrfRWYqKqTgc3AXb1dVDse4eP1IiKZwPnArmAcNKzDApgJFKrqNlWtBZYD80NcU5tUdZ+qrnFfH8H5MhsR2qraJiIZwKeA34e6lo6ISCJwFvAHAFWtVdXDoa2qXRFArIhEAIOAvSGu5wSq+h+grMXi+cAf3dd/BD7dq0W1obVaVfUfqlrvvn0XyOj1wtrQxu8W4OfAN4CgnLUU7mExAtjt976IPvzl609EsoBpwHuhraRdS3H+520MdSGdkA0UAw+73Wa/F5G4UBfVGlXdA/wU5y/IfUC5qv4jtFV1Sqqq7nNf7wdSQ1lMF9wAvBzqItojIvOBPar6QbCOEe5h0S+JSDzwNPAVVa0IdT2tEZGLgYOqujrUtXRSBDAd+I2qTgOO0ne6SU7g9vXPxwm4dCBORK4JbVVdo845+33+vH0R+RZO9++fQ11LW0RkEPBN4O5gHifcw2IPkOn3PsNd1meJSCROUPxZVf8W6nracQYwT0R24HTvnSMij4W2pHYVAUWq2tRSewonPPqi84DtqlqsqnXA34DTQ1xTZxwQkeEA7vPBENfTLhG5DrgYuFr79gVpo3H+cPjA/feWAawRkbRAHiTcw2IVMEZEskUkCmeQcEWIa2qTiAhOn/oGVV0S6nrao6p3qWqGqmbh/F7/pap99q9fVd0P7BaRHHfRucD6EJbUnl3ALBEZ5P4/cS59dDC+hRXAte7ra4HnQlhLu0RkLk4X6jxVrQp1Pe1R1Q9VdZiqZrn/3oqA6e7/0wET1mHhDmDdCryC84/tSVVdF9qq2nUG8Dmcv9LXuo+LQl3UAPJl4M8iUgBMBX4Q4npa5bZ+ngLWAB/i/DvuU1NTiMgTwDtAjogUiciNwH3AJ0VkC07r6L5Q1tikjVp/BfiAV91/Zw+GtEg/bdQb/OP27daVMcaYviCsWxbGGGM6x8LCGGNMhywsjDHGdMjCwhhjTIcsLIwxxnTIwsKYLhCRBr/TltcGcqZiEclqbSZRY/qCiFAXYEw/c0xVp4a6CGN6m7UsjAkAEdkhIj8WkQ9FZKWInOwuzxKRf7n3RXhNREa6y1Pd+yR84D6apuvwisjv3HtV/ENEYkP2Qxnjx8LCmK6JbdENdaXfZ+WqOgnn6t+l7rJfAn9074vwZ+AX7vJfAG+o6hScOaiaZg4YAzygqhOAw8DlQf55jOkUu4LbmC4QkUpVjW9l+Q7gHFXd5k72uF9Vh4hICTBcVevc5ftUdaiIFAMZqlrjt48s4FX35kCIyP8Bkar6/eD/ZMa0z1oWxgSOtvG6K2r8Xjdg44qmj7CwMCZwrvR7fsd9/TbHb3l6NfCm+/o14BZovk95Ym8VaUx32F8txnRNrIis9Xv/d1VtOn12sDtjbQ2w0F32ZZy7730d505817vLbwcecmcMbcAJjn0Y00fZmIUxAeCOWeSpakmoazEmGKwbyhhjTIesZWGMMaZD1rIwxhjTIQsLY4wxHbKwMMYY0yELC2OMMR2ysDDGGNOh/w/DonLhx2CFWQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluation = model.evaluate(x = X_test, y = y_test, verbose=1) \n",
    "loss = evaluation[0]\n",
    "acc = evaluation[1]\n",
    "print('LOSS ON TEST DATA='+str(loss))\n",
    "print('ACCURACY ON TEST DATA='+str(acc))\n",
    "\n",
    "\n",
    "# Plot training & validation loss values\n",
    "plt.plot(history.history['loss'], label='Train')\n",
    "plt.plot(history.history['val_loss'], label='Validation')\n",
    "plt.title('Model loss')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Validation'], loc='upper left')\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.98      0.98       476\n",
      "           1       0.92      0.94      0.93       103\n",
      "\n",
      "   micro avg       0.97      0.97      0.97       579\n",
      "   macro avg       0.95      0.96      0.96       579\n",
      "weighted avg       0.97      0.97      0.97       579\n",
      "\n",
      "Accuracy: 0.9740932642487047\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAF4dJREFUeJzt3X20XXV95/H3xwREBYMasJoEgjxUqU9oCjiupcyIFBkNLusDGdHiUNJaaXW0ts6qo5RWrTrasQWrVCiKCoJjXWmNxSc0oxVNHJACCo3IQwIOIBBEMDx954+9Lzlekn1Prnffc254v9a6i7P3+Z29v/vHzfncvX/n/HaqCkmStuVhoy5AkjTeDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0JzSpLLkhw2RZu9ktyRZN4sldW7JFcnObx9fFKST466Jj10GBSaEe0b2V3tG/T/S3Jmkl1nej9V9RtV9fUp2lxbVbtW1X0zvf/2Tfqe9jhvS/KvSZ4z0/uRxolBoZn0kqraFXgWsAx4++QGacz137vPtMe5ELgAOG/E9cy4JPNHXYPGx1z/B6sxVFUbgS8CTwVI8vUk70ryLeBO4ElJFiQ5PckNSTYm+cvBS0VJTkjygyQ/S3J5kme16wcvwRycZF2S29uzmA+265cmqYk3uyRPTLIqyS1J1ic5YWA/JyU5N8kn2n1dlmTZkMd5L/ApYFGSPQa2+eIkFw+ccTx94LklST6X5KYkP01ySrt+3yRfa9fdnORTSXafTv8nObrd/+1JfpTkyMl9N3Dsn5zUZ8cnuRb4WpIvJjlx0ra/n+Rl7eMnJ/ly269XJHnldOrV+DMoNOOSLAGOAi4aWP0aYCWwG3ANcCZwL7AfcBBwBPC77etfAZwEvBZ4NLAc+OlWdvUh4ENV9WhgX+DcbZR0DrABeCLwcuDdSf7TwPPL2za7A6uAU4Y8zp3bGn8K3NquOwg4A/g94HHAR4FVSR7eBuE/t8e/FFjU7hcgwHvaGp8CLGn7YLskORj4BPDW9nieB1y9HZt4frv/3wLOBlYMbPtAYG/gC0keBXwZ+DSwJ3AM8OG2jXYwBoVm0ueT3AZ8E/gG8O6B586sqsvav8IfSxMkb6qqn1fVjcBf07zZQBMY76uqtdVYX1XXbGV/9wD7JVlYVXdU1YWTG7Sh9VzgT6vqF1V1MfAxmjf4Cd+sqtXtmMZZwDOmOM5Xtsd5F3AC8PL2uKAJw49W1Xeq6r6q+jiwGTgUOJgmCN7aHvcvquqbAO0xfrmqNlfVTcAHad60t9fxwBnttu6vqo1V9cPteP1JbW13Af8IPDPJ3u1zrwY+V1WbgRcDV1fVP1TVvVV1EfC/gVdMo2aNOYNCM+mlVbV7Ve1dVX/QvtlMuG7g8d7ATsAN7eWZ22j+8t6zfX4J8KMh9nc8cADwwyRrk7x4K22eCNxSVT8bWHcNzV/zE34y8PhOYJck85O8uh20viPJFwfanFtVuwOPBy4Fnj3p2N4ycVztsS1p61gCXDMQKg9I8vgk57SX4W4HPkkzBrK9hu27bXng/1PbZ19gS4CvoLnUBs1xHjLpOF8N/NqvsG+NKQesNFsGpym+juav7IVbe9Nsn993yg1W/Tuwoh0cfxnw2SSPm9TseuCxSXYbCIu9gI1DbP9TbHlj3NrzNydZCaxL8umquqGt/V1V9a7J7dtPR+2VZP5WjvvdNH30tKq6JclLGfIS2CRdffdz4JEDy1t7U588nfTZwDuTrAF2oRm8n9jPN6rqhdOoUXOMZxSade0b6peADyR5dJKHtYO5E5daPgb8cZJnNx+Syn4Dlz8ekOTYJHtU1f3Abe3q+yft6zrgX4H3JNmlHVg+nuYv9pk4liuA84E/aVf9PfD7SQ5pa39Ukv+cZDfgu8ANwF+163dJ8tz2dbsBdwCbkiyiGWOYjtOB1yV5Qduvi5I8uX3uYuCYJDu1A/YvH2J7q2nOHk6m+bTXRP/+M3BAkte029spyW8meco069YYMyg0Kq8FdgYupxkI/izwBICqOg94F81A6c+Az9OMa0x2JHBZkjtoBraPmXS5a8IKmsHj62muu7+zqr4yg8fyfmBlkj2rah3NuMUp7XGtB44DaMdAXkIzgH8tzQD7q9pt/DnNx4o30Vzu+dx0Cqmq7wKvoxnz2UQzVjQRsv+D5mzj1nZ/nx5ie5vbWg4fbN+enR1Bc1nqeprLd+8FHj6dujXe4o2LJEldPKOQJHUyKCRJnQwKSVIng0KS1GnOfY9i4cKFtXTp0lGXIUlzyve+972bq2qPqVs+2JwLiqVLl7Ju3bpRlyFJc0qSrU2DMxQvPUmSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTr0FRZIzktyY5NJtPJ8kf5PmHsaXpL0nsiRpvPR5RnEmzTTQ2/IiYP/2ZyXwdz3WIkmapt6CoqrWALd0NDka+ER7T+QLgd2TPGGq7d5990xVKEkaxijHKBbxy/dR3sAv38f4AUlWJlmXZN0NN9w6K8VJkhpzYjC7qk6rqmVVtWzBgseMuhxJekgZZVBsBJYMLC9miBveS5Jm1yiDYhXw2vbTT4cCm6rqhhHWI0nait5mj01yNnAYsDDJBuCdwE4AVfURYDVwFM3N5++kuSG8JGnM9BYUVbViiucLeENf+5ckzYw5MZgtSRodg0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUqbcpPPpy//1w5ZWjrkKS+vXYx8LChaOuojHnggJgzZpRVyBJ/dm8uQmKFZ0z5s2eORcU8+fDQQeNugpJ6s8118AtXTeSnmWOUUiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKlTr0GR5MgkVyRZn+RtW3l+ryQXJLkoySVJjuqzHknS9ustKJLMA04FXgQcCKxIcuCkZm8Hzq2qg4BjgA/3VY8kaXr6PKM4GFhfVVdV1d3AOcDRk9oU8Oj28QLg+h7rkSRNw/wet70IuG5geQNwyKQ2JwFfSvKHwKOAw7e2oSQrgZUAe+6514wXKknatlEPZq8AzqyqxcBRwFlJHlRTVZ1WVcuqatmCBXvMepGS9FDWZ1BsBJYMLC9u1w06HjgXoKq+DewCLOyxJknSduozKNYC+yfZJ8nONIPVqya1uRZ4AUCSp9AExU091iRJ2k69BUVV3QucCJwP/IDm002XJTk5yfK22VuAE5J8HzgbOK6qqq+aJEnbr8/BbKpqNbB60rp3DDy+HHhunzVIkn41ox7MliSNOYNCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHWaP2zDJIuAvQdfU1Vr+ihKkjQ+hgqKJO8FXgVcDtzXri6gMyiSHAl8CJgHfKyq/morbV4JnNRu7/tV9V+GLV6S1L9hzyheCvx6VW0edsNJ5gGnAi8ENgBrk6yqqssH2uwP/HfguVV1a5I9hy9dkjQbhh2juArYaTu3fTCwvqquqqq7gXOAoye1OQE4tapuBaiqG7dzH5Kkng17RnEncHGSrwIPnFVU1R91vGYRcN3A8gbgkEltDgBI8i2ay1MnVdW/DFmTJGkWDBsUq9qfPva/P3AYsBhYk+RpVXXbYKMkK4GVAHvuuVcPZUiStmWooKiqjyfZmfYMALiiqu6Z4mUbgSUDy4vbdYM2AN9pt/XjJFfSBMfaSfs/DTgN4IADltUwNUuSZsZQYxRJDgP+nWZw+sPAlUmeN8XL1gL7J9mnDZljePBZyedpziZIspAmiK4atnhJUv+GvfT0AeCIqroCIMkBwNnAs7f1gqq6N8mJwPk04w9nVNVlSU4G1lXVqva5I5JMfOz2rVX10+kfjiRppg0bFDtNhARAVV2ZZMpPQVXVamD1pHXvGHhcwJvbH0nSGBo2KNYl+RjwyXb51cC6fkqSJI2TYYPi9cAbgImPw/4fmrEKSdIObthPPW0GPtj+SJIeQjqDIsm5VfXKJP9GMxfTL6mqp/dWmSRpLEx1RvHG9r8v7rsQSdJ46vweRVXd0D68Gbiuqq4BHg48A7i+59okSWNg2EkB1wC7tPek+BLwGuDMvoqSJI2PYYMiVXUn8DLgw1X1CuA3+itLkjQuhg6KJM+h+f7EF9p18/opSZI0ToYNijfR3GDoH9tpOJ4EXNBfWZKkcTHs9yi+AXxjYPkqtnz5TpK0A5vqexT/q6relOSf2Pr3KJb3VpkkaSxMdUZxVvvf/9l3IZKk8dQZFFX1vfbhOuCuqrofIMk8mu9TSJJ2cMMOZn8VeOTA8iOAr8x8OZKkcTNsUOxSVXdMLLSPH9nRXpK0gxg2KH6e5FkTC0meDdzVT0mSpHEy7P0o3gScl+R6IMCvAa/qrSpJ0tgY9nsUa5M8Gfj1dtUVVXVPf2VJksbFUJeekjwS+FPgjVV1KbA0iVOPS9JDwLBjFP8A3A08p13eCPxlLxVJksbKsEGxb1W9D7gHoJ1JNr1VJUkaG8MGxd1JHkE7jUeSfYHNvVUlSRobw37q6Z3AvwBLknwKeC5wXF9FSZLGx5RBkSTAD2luWnQozSWnN1bVzT3XJkkaA1MGRVVVktVV9TS23LRIkvQQMewYxf9N8pu9ViJJGkvDjlEcAhyb5Grg5zSXn6qqnt5XYZKk8TBsUPxWr1VIksbWVHe42wX4fWA/4N+A06vq3tkoTJI0HqYao/g4sIwmJF4EfKD3iiRJY2WqS08Htp92IsnpwHf7L0mSNE6mOqN4YIZYLzlJ0kPTVEHxjCS3tz8/A54+8TjJ7VNtPMmRSa5Isj7J2zra/XaSSrJsew9AktSvzktPVTVvuhtOMg84FXghsAFYm2RVVV0+qd1uwBuB70x3X5Kk/gz7hbvpOBhYX1VXVdXdwDnA0Vtp9xfAe4Ff9FiLJGma+gyKRcB1A8sb2nUPaO/DvaSqOqcGSbIyybok6zZtumnmK5UkbVOfQdEpycOADwJvmaptVZ1WVcuqatmCBXv0X5wk6QF9BsVGYMnA8uJ23YTdgKcCX2+nBjkUWOWAtiSNlz6DYi2wf5J9kuwMHAOsmniyqjZV1cKqWlpVS4ELgeVVta7HmiRJ26m3oGi/d3EicD7wA+DcqrosyclJlve1X0nSzBp2UsBpqarVwOpJ696xjbaH9VmLJGl6RjaYLUmaGwwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdeo1KJIcmeSKJOuTvG0rz785yeVJLkny1SR791mPJGn79RYUSeYBpwIvAg4EViQ5cFKzi4BlVfV04LPA+/qqR5I0PX2eURwMrK+qq6rqbuAc4OjBBlV1QVXd2S5eCCzusR5J0jT0GRSLgOsGlje067bleOCLW3siycok65Ks27TpphksUZI0lbEYzE5yLLAMeP/Wnq+q06pqWVUtW7Bgj9ktTpIe4ub3uO2NwJKB5cXtul+S5HDgz4DnV9XmHuuRJE1Dn2cUa4H9k+yTZGfgGGDVYIMkBwEfBZZX1Y091iJJmqbeziiq6t4kJwLnA/OAM6rqsiQnA+uqahXNpaZdgfOSAFxbVcv7qkmS5orNm+HKK0ddRaPPS09U1Wpg9aR17xh4fHif+5ekuWjBAvjJT2DNmpnc6m6Pmu4rew0KSdL22313OOSQmd7qvHnTfeVYfOpJkjS+DApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR16jUokhyZ5Iok65O8bSvPPzzJZ9rnv5NkaZ/1SJK2X29BkWQecCrwIuBAYEWSAyc1Ox64tar2A/4aeG9f9UiSpqfPM4qDgfVVdVVV3Q2cAxw9qc3RwMfbx58FXpAkPdYkSdpO83vc9iLguoHlDcAh22pTVfcm2QQ8Drh5sFGSlcDKdumeZcsec3UvFc85mxfAwzeNuorxYF9sYV9sYV9scfvi6b6yz6CYMVV1GnAaQJJ1VbcuG3FJY6HpizvtC+yLQfbFFvbFFknWTfe1fV562ggsGVhe3K7bapsk84EFwE97rEmStJ36DIq1wP5J9kmyM3AMsGpSm1XA77SPXw58raqqx5okSdupt0tP7ZjDicD5wDzgjKq6LMnJwLqqWgWcDpyVZD1wC02YTOW0vmqeg+yLLeyLLeyLLeyLLabdF/EPeElSF7+ZLUnqZFBIkjqNbVA4/ccWQ/TFm5NcnuSSJF9Nsvco6pwNU/XFQLvfTlJJdtiPRg7TF0le2f5uXJbk07Nd42wZ4t/IXkkuSHJR++/kqFHU2bckZyS5Mcml23g+Sf6m7adLkjxrqA1X1dj90Ax+/wh4ErAz8H3gwElt/gD4SPv4GOAzo657hH3xH4FHto9f/1Dui7bdbsAa4EJg2ajrHuHvxf7ARcBj2uU9R133CPviNOD17eMDgatHXXdPffE84FnApdt4/ijgi0CAQ4HvDLPdcT2jcPqPLabsi6q6oKrubBcvpPnOyo5omN8LgL+gmTfsF7NZ3Cwbpi9OAE6tqlsBqurGWa5xtgzTFwU8un28ALh+FuubNVW1huYTpNtyNPCJalwI7J7kCVNtd1yDYmvTfyzaVpuquheYmP5jRzNMXww6nuYvhh3RlH3RnkovqaovzGZhIzDM78UBwAFJvpXkwiRHzlp1s2uYvjgJODbJBmA18IezU9rY2d73E2COTOGh4SQ5FlgGPH/UtYxCkocBHwSOG3Ep42I+zeWnw2jOMtckeVpV3TbSqkZjBXBmVX0gyXNovr/11Kq6f9SFzQXjekbh9B9bDNMXJDkc+DNgeVVtnqXaZttUfbEb8FTg60muprkGu2oHHdAe5vdiA7Cqqu6pqh8DV9IEx45mmL44HjgXoKq+DewCLJyV6sbLUO8nk41rUDj9xxZT9kWSg4CP0oTEjnodGqboi6raVFULq2ppVS2lGa9ZXlXTngxtjA3zb+TzNGcTJFlIcynqqtkscpYM0xfXAi8ASPIUmqC4aVarHA+rgNe2n346FNhUVTdM9aKxvPRU/U3/MecM2RfvB3YFzmvH86+tquUjK7onQ/bFQ8KQfXE+cESSy4H7gLdW1Q531j1kX7wF+Psk/41mYPu4HfEPyyRn0/xxsLAdj3knsBNAVX2EZnzmKGA9cCfwuqG2uwP2lSRpBo3rpSdJ0pgwKCRJnQwKSVIng0KS1MmgkCR1MiikSZLcl+TiJJcm+acku8/w9o9Lckr7+KQkfzyT25dmmkEhPdhdVfXMqnoqzXd03jDqgqRRMiikbt9mYNK0JG9Nsrady//PB9a/tl33/SRntete0t4r5aIkX0ny+BHUL/3KxvKb2dI4SDKPZtqH09vlI2jmSjqYZj7/VUmeRzPH2NuB/1BVNyd5bLuJbwKHVlUl+V3gT2i+ISzNKQaF9GCPSHIxzZnED4Avt+uPaH8uapd3pQmOZwDnVdXNAFU1cT+AxcBn2vn+dwZ+PDvlSzPLS0/Sg91VVc8E9qY5c5gYowjwnnb84plVtV9Vnd6xnb8FTqmqpwG/RzMRnTTnGBTSNrR3Dfwj4C3tVPbnA/81ya4ASRYl2RP4GvCKJI9r109celrAlimcfwdpjvLSk9Shqi5KcgmwoqrOaqeo/nY7S+8dwLHtTKXvAr6R5D6aS1PH0dxV7bwkt9KEyT6jOAbpV+XssZKkTl56kiR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUqf/DwxLCVM+P7IQAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluate(model, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pretrained Glove word embeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download link:\n",
    "http://nlp.stanford.edu/data/glove.6B.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_NB_WORDS = 100000\n",
    "MAX_SEQUENCE_LENGTH = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/spiros/env/lib/python3.5/site-packages/keras_preprocessing/text.py:177: UserWarning: The `nb_words` argument in `Tokenizer` has been renamed `num_words`.\n",
      "  warnings.warn('The `nb_words` argument in `Tokenizer` '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 60960 unique tokens.\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "tokenizer = Tokenizer(nb_words=MAX_NB_WORDS)\n",
    "tokenizer.fit_on_texts(mails)\n",
    "sequences = tokenizer.texts_to_sequences(mails)\n",
    "\n",
    "word_index = tokenizer.word_index\n",
    "print('Found %s unique tokens.' % len(word_index))\n",
    "\n",
    "data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "GLOVE_DIR = '/media/spiros/Data/glove.6B/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 400000 word vectors.\n"
     ]
    }
   ],
   "source": [
    "embeddings_index = {}\n",
    "f = open(os.path.join(GLOVE_DIR, 'glove.6B.300d.txt'))\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:], dtype='float32')\n",
    "    embeddings_index[word] = coefs\n",
    "f.close()\n",
    "\n",
    "print('Found %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Difference between embedding and dataset tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21955"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mail_tokens = set(tokenizer.word_index.keys())\n",
    "embedding_tokens = set(embeddings_index.keys())\n",
    "\n",
    "len(list(mail_tokens - embedding_tokens))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMBEDDING_DIM = 300\n",
    "embedding_matrix = np.zeros((len(word_index) + 1, EMBEDDING_DIM))\n",
    "for word, i in word_index.items():\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Embedding\n",
    "\n",
    "embedding_layer = Embedding(len(word_index) + 1,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(data, y, test_size=0.2, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_up(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 100, 300)          18288300  \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 100, 300)          0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 30000)             0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1000)              30001000  \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 1000)              0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 1000)              4000      \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 500)               500500    \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 500)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 500)               2000      \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 100)               50100     \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 100)               400       \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 48,846,401\n",
      "Trainable params: 30,554,901\n",
      "Non-trainable params: 18,291,500\n",
      "_________________________________________________________________\n",
      "Train on 2082 samples, validate on 232 samples\n",
      "Epoch 1/30\n",
      "2082/2082 [==============================] - 7s 3ms/step - loss: 0.5490 - acc: 0.7128 - val_loss: 0.2293 - val_acc: 0.9181\n",
      "Epoch 2/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.2630 - acc: 0.8905 - val_loss: 0.1445 - val_acc: 0.9526\n",
      "Epoch 3/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.1885 - acc: 0.9299 - val_loss: 0.1529 - val_acc: 0.9569\n",
      "Epoch 4/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.1156 - acc: 0.9544 - val_loss: 0.1682 - val_acc: 0.9397\n",
      "Epoch 5/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.1045 - acc: 0.9625 - val_loss: 0.1285 - val_acc: 0.9612\n",
      "Epoch 6/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0919 - acc: 0.9702 - val_loss: 0.1446 - val_acc: 0.9526\n",
      "Epoch 7/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.1001 - acc: 0.9616 - val_loss: 0.1928 - val_acc: 0.9353\n",
      "Epoch 8/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0908 - acc: 0.9673 - val_loss: 0.1581 - val_acc: 0.9483\n",
      "Epoch 9/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0685 - acc: 0.9745 - val_loss: 0.1804 - val_acc: 0.9483\n",
      "Epoch 10/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0878 - acc: 0.9669 - val_loss: 0.1762 - val_acc: 0.9440\n",
      "Epoch 11/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0604 - acc: 0.9755 - val_loss: 0.1753 - val_acc: 0.9483\n",
      "Epoch 12/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0687 - acc: 0.9741 - val_loss: 0.1634 - val_acc: 0.9397\n",
      "Epoch 13/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0613 - acc: 0.9731 - val_loss: 0.1811 - val_acc: 0.9353\n",
      "Epoch 14/30\n",
      "2082/2082 [==============================] - 5s 3ms/step - loss: 0.0659 - acc: 0.9765 - val_loss: 0.1971 - val_acc: 0.9310\n",
      "Epoch 15/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0572 - acc: 0.9798 - val_loss: 0.1878 - val_acc: 0.9397\n",
      "Epoch 16/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0532 - acc: 0.9793 - val_loss: 0.1871 - val_acc: 0.9397\n",
      "Epoch 17/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0474 - acc: 0.9851 - val_loss: 0.2124 - val_acc: 0.9353\n",
      "Epoch 18/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0376 - acc: 0.9885 - val_loss: 0.2218 - val_acc: 0.9397\n",
      "Epoch 19/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0389 - acc: 0.9885 - val_loss: 0.2455 - val_acc: 0.9353\n",
      "Epoch 20/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0327 - acc: 0.9861 - val_loss: 0.2304 - val_acc: 0.9397\n",
      "Epoch 21/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0419 - acc: 0.9866 - val_loss: 0.2349 - val_acc: 0.9353\n",
      "Epoch 22/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0400 - acc: 0.9851 - val_loss: 0.2647 - val_acc: 0.9267\n",
      "Epoch 23/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0474 - acc: 0.9808 - val_loss: 0.2158 - val_acc: 0.9397\n",
      "Epoch 24/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0455 - acc: 0.9841 - val_loss: 0.2037 - val_acc: 0.9440\n",
      "Epoch 25/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0490 - acc: 0.9808 - val_loss: 0.2031 - val_acc: 0.9440\n",
      "Epoch 26/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0411 - acc: 0.9856 - val_loss: 0.1597 - val_acc: 0.9569\n",
      "Epoch 27/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0443 - acc: 0.9846 - val_loss: 0.1823 - val_acc: 0.9526\n",
      "Epoch 28/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0363 - acc: 0.9875 - val_loss: 0.2202 - val_acc: 0.9353\n",
      "Epoch 29/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0297 - acc: 0.9890 - val_loss: 0.1918 - val_acc: 0.9526\n",
      "Epoch 30/30\n",
      "2082/2082 [==============================] - 5s 2ms/step - loss: 0.0307 - acc: 0.9918 - val_loss: 0.1935 - val_acc: 0.9526\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "\n",
    "model.add(embedding_layer)\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(1000, kernel_initializer='glorot_normal', activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dense(500, kernel_initializer='glorot_normal', activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dense(100, kernel_initializer='glorot_normal', activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dense(1, kernel_initializer='glorot_normal', activation='sigmoid'))\n",
    "model.summary()\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "history = model.fit(X_train, y_train, epochs=30, batch_size=16, verbose=1, validation_split=0.1)\n",
    "\n",
    "y_pred = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "579/579 [==============================] - 0s 76us/step\n",
      "LOSS ON TEST DATA=0.0858191350552966\n",
      "ACCURACY ON TEST DATA=0.9723661485319517\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl4VNX5wPHvm30nO0sSCAKybyGiCMiiIqKCKC641b1Srbb+tLV20Vqt2lqLW23RqrUuuCCKinVFBRdWAWVHCBC2LJBA9szM+f1xJiFAlpmQYZLM+3mePJm5c++dc5lw3znbe8QYg1JKKQUQ5O8CKKWUaj00KCillKqlQUEppVQtDQpKKaVqaVBQSilVS4OCUkqpWhoUlPKAiGSKiBGREA/2vVpEFh3reZTyBw0Kqt0RkRwRqRKR5CO2f+e+IWf6p2RKtX4aFFR7tRWYXvNERAYCUf4rjlJtgwYF1V79F7iqzvOfAC/W3UFEOojIiyKSLyLbROR3IhLkfi1YRB4RkQIR2QKcU8+x/xaR3SKyU0TuF5FgbwspIl1EZJ6I7BORzSJyQ53XhovIMhE5ICJ7ReRR9/YIEXlJRApFpEhElopIR2/fW6n6aFBQ7dW3QJyI9HXfrC8FXjpinyeADsAJwBhsELnG/doNwLnAUCAbmHbEsS8ADqCne58JwPXNKOdsIBfo4n6PP4vIePdrjwGPGWPigB7A6+7tP3GXOwNIAm4Cypvx3kodRYOCas9qagtnAuuAnTUv1AkUvzHGHDTG5AB/A65073IxMNMYs8MYsw94sM6xHYFJwC+MMaXGmDzg7+7zeUxEMoCRwK+NMRXGmJXAsxyq4VQDPUUk2RhTYoz5ts72JKCnMcZpjFlujDngzXsr1RANCqo9+y9wGXA1RzQdAclAKLCtzrZtQJr7cRdgxxGv1ejmPna3u/mmCPgXkOpl+boA+4wxBxsow3XAicB6dxPRuXWu60NgtojsEpG/iEiol++tVL00KKh2yxizDdvhPAl464iXC7DfuLvV2daVQ7WJ3djmmbqv1dgBVALJxph490+cMaa/l0XcBSSKSGx9ZTDGbDLGTMcGm4eBN0Uk2hhTbYz5ozGmH3AqtpnrKpRqARoUVHt3HTDeGFNad6Mxxolto39ARGJFpBtwO4f6HV4HbhWRdBFJAO6qc+xu4CPgbyISJyJBItJDRMZ4UzBjzA7ga+BBd+fxIHd5XwIQkStEJMUY4wKK3Ie5RGSciAx0N4EdwAY3lzfvrVRDNCiods0Y86MxZlkDL/8cKAW2AIuAV4Dn3K89g22iWQWs4OiaxlVAGLAW2A+8CXRuRhGnA5nYWsNc4B5jzCfu1yYCa0SkBNvpfKkxphzo5H6/A9i+ki+wTUpKHTPRRXaUUkrV0JqCUkqpWhoUlFJK1dKgoJRSqpYGBaWUUrXaXPre5ORkk5mZ6e9iKKVUm7J8+fICY0xKU/u1uaCQmZnJsmUNjTBUSilVHxHZ1vRe2nyklFKqDg0KSimlamlQUEopVavN9SnUp7q6mtzcXCoqKvxdlHYjIiKC9PR0QkM1+aZSgaRdBIXc3FxiY2PJzMxERPxdnDbPGENhYSG5ubl0797d38VRSh1H7aL5qKKigqSkJA0ILURESEpK0pqXUgGoXQQFQANCC9N/T6UCU7sJCk0prXSwu7gczQqrlFINC5igUFblJP9gJU5XyweFwsJChgwZwpAhQ+jUqRNpaWm1z6uqqjw6xzXXXMOGDRtavGxKKeWNdtHR7InQYNsc4nAZQoJb9txJSUmsXLkSgHvvvZeYmBjuuOOOw/YxxmCMISio/jj8/PPPt2yhlFKqGQKmphASbC+12nn8Vi3cvHkz/fr14/LLL6d///7s3r2bG2+8kezsbPr37899991Xu++oUaNYuXIlDoeD+Ph47rrrLgYPHsyIESPIy8s7bmVWSgW2dldT+OO7a1i768BR240xlFU5CQ8NJiTIu07Ufl3iuOc8b9dkt9avX8+LL75IdnY2AA899BCJiYk4HA7GjRvHtGnT6Nev32HHFBcXM2bMGB566CFuv/12nnvuOe666676Tq+UUi0qYGoKNaNpjndHc48ePWoDAsCrr75KVlYWWVlZrFu3jrVr1x51TGRkJGeffTYAw4YNIycn53gVVykV4NpdTaGxb/Q/7CwmMTqMLvGRx6080dHRtY83bdrEY489xpIlS4iPj+eKK66ody5AWFhY7ePg4GAcDsdxKatSSgVMTQEgJFhwOP03JPXAgQPExsYSFxfH7t27+fDDD/1WFqWUqk+7qyk0JjQoiGrX8etoPlJWVhb9+vWjT58+dOvWjZEjR/qtLEopVR9pa5O5srOzzZGL7Kxbt46+ffs2eey2wlIqql307hTrq+K1K57+uyqlWj8RWW6MyW5qv4BqPgoNDsJxHIekKqVUWxNQQSEkSHAag8sHs5qVUqo9CKygUDOBzY/9Ckop1ZoFVFCoTXXhxxFISinVmgVUUAhx5x3SfgWllKpfYAUFd02hWvsUlFKqXoEVFIIEQVq8pjBu3LijJqLNnDmTGTNmNHhMTEwMALt27WLatGn17jN27FiOHH57pJkzZ1JWVlb7fNKkSRQVFXladKWUOoxPg4KITBSRDSKyWUSOyugmIleLSL6IrHT/XO/j8vhkVvP06dOZPXv2Ydtmz57N9OnTmzy2S5cuvPnmm81+7yODwvz584mPj2/2+ZRSgc1nQUFEgoGngLOBfsB0EelXz66vGWOGuH+e9VV5aoQESYs3H02bNo3333+/dkGdnJwcdu3axdChQzn99NPJyspi4MCBvPPOO0cdm5OTw4ABAwAoLy/n0ksvpW/fvkydOpXy8vLa/WbMmFGbcvuee+4B4PHHH2fXrl2MGzeOcePGAZCZmUlBQQEAjz76KAMGDGDAgAHMnDmz9v369u3LDTfcQP/+/ZkwYcJh76OUCmy+THMxHNhsjNkCICKzgSnA0WlBW9IHd8Ge7xt8OaPaiQsDoV5ceqeBcPZDDb6cmJjI8OHD+eCDD5gyZQqzZ8/m4osvJjIykrlz5xIXF0dBQQGnnHIKkydPbnD946effpqoqCjWrVvH6tWrycrKqn3tgQceIDExEafTyemnn87q1au59dZbefTRR1mwYAHJycmHnWv58uU8//zzLF68GGMMJ598MmPGjCEhIYFNmzbx6quv8swzz3DxxRczZ84crrjiCs//PZRS7ZYvm4/SgB11nue6tx3pQhFZLSJvikhGfScSkRtFZJmILMvPzz+mQomALzJ71G1Cqmk6MsZw9913M2jQIM444wx27tzJ3r17GzzHl19+WXtzHjRoEIMGDap97fXXXycrK4uhQ4eyZs2aelNu17Vo0SKmTp1KdHQ0MTExXHDBBSxcuBCA7t27M2TIEEBTcyulDufvhHjvAq8aYypF5KfAf4DxR+5kjJkFzAKb+6jRMzbyjR5gf3EF+QcrGJDWocFv7M0xZcoUfvnLX7JixQrKysoYNmwYL7zwAvn5+SxfvpzQ0FAyMzPrTZXdlK1bt/LII4+wdOlSEhISuPrqq5t1nhrh4eG1j4ODg7X5SClVy5c1hZ1A3W/+6e5ttYwxhcaYSvfTZ4FhPiwPYCewGexazS0pJiaGcePGce2119Z2MBcXF5OamkpoaCgLFixg27ZtjZ7jtNNO45VXXgHghx9+YPXq1YBNuR0dHU2HDh3Yu3cvH3zwQe0xsbGxHDx48KhzjR49mrfffpuysjJKS0uZO3cuo0ePbqnLVUq1U76sKSwFeolId2wwuBS4rO4OItLZGLPb/XQysM6H5QEOpbpwOF2EBrdsTJw+fTpTp06tbUa6/PLLOe+88xg4cCDZ2dn06dOn0eNnzJjBNddcQ9++fenbty/DhtkYOXjwYIYOHUqfPn3IyMg4LOX2jTfeyMSJE+nSpQsLFiyo3Z6VlcXVV1/N8OHDAbj++usZOnSoNhUppRrl09TZIjIJmAkEA88ZYx4QkfuAZcaYeSLyIDYYOIB9wAxjzPrGznksqbMBSisd/JhfQmZSNHGRod5fVADR1NlKtR+eps72aZ+CMWY+MP+IbX+o8/g3wG98WYYj1eY/0qR4Sil1lICa0Qx18x9pqgullDpSuwkKnjaDBQUJwT6YwNbetLUV+ZRSLaNdBIWIiAgKCws9vpGFBukKbI0xxlBYWEhERIS/i6KUOs78PU+hRaSnp5Obm4unE9sKDlZigLK88Cb3DVQRERGkp6f7uxhKqeOsXQSF0NBQunfv7vH+v5j9Hcu372fhr46aJ6eUUgGtXTQfeSs1LoK8A5Xabq6UUkcIyKCQEhNOpcPFgQqHv4uilFKtSkAGhdQ425eQf7CyiT2VUiqwBGRQSIm1QSHvYPOTyimlVHsUkEEhNVZrCkopVZ+ADAopsXb8vQYFpZQ6XEAGhbiIEMJDgsjToKCUUocJyKAgIqTEhpN3QPsUlFKqroAMCmD7FfJLtKaglFJ1BXBQsBPYlFJKHRKwQSElNlz7FJRS6ggBGxRSY8MpLq+m0uH0d1GUUqrVCNygoLOalVLqKAEbFA7NatagoJRSNQI2KKTqBDallDpKAAcFrSkopdSRAjYoJEaHIQL5OoFNKaVqBWxQCAkOIilah6UqpVRdARsUwD2rWYOCUkrVCuygEKc1BaWUqiugg0JKTLgutKOUUnUEdFBIjQunoKQKl8v4uyhKKdUqBHZQiI3A6TLsK6vyd1GUUqpVCOigUDurWbOlKqUU4OOgICITRWSDiGwWkbsa2e9CETEiku3L8hypdq1mXVdBKaUAHwYFEQkGngLOBvoB00WkXz37xQK3AYt9VZaG1KS60BXYlFLK8mVNYTiw2RizxRhTBcwGptSz35+Ah4HjfmfWpHhKKXU4XwaFNGBHnee57m21RCQLyDDGvN/YiUTkRhFZJiLL8vPzW6yAkWHBxIaH6AQ2pZRy81tHs4gEAY8C/9fUvsaYWcaYbGNMdkpKSouWIyVOZzUrpVQNXwaFnUBGnefp7m01YoEBwOcikgOcAsw73p3NOoFNKaUO8WVQWAr0EpHuIhIGXArMq3nRGFNsjEk2xmQaYzKBb4HJxphlPizTUVLjIrSmoJRSbj4LCsYYB3AL8CGwDnjdGLNGRO4Tkcm+el9vpcZq/iOllKoR4suTG2PmA/OP2PaHBvYd68uyNCQlNpyyKicllQ5iwn36z6GUUq1eQM9ohjorsOlcBaWU0qCgazUrpdQhAR8UdAKbUkodEvBBIVWDglJK1Qr4oBAfFUposGjzkVJKoUEBEdEJbEop5RbwQQEgRSewKaUUoEEBsP0KGhSUUkqDAqCzmpVSqoYGBeyw1H2lVVQ5XP4uilJK+ZUGBQ5NYCss1dqCUiqwaVCgbqoLDQpKqcCmQQGd1ayUUjU0KACpcTYo6AgkpVSg06AAJMfU1BR0AptSKrBpUABCg4NIjA7T5iOlVMDToOCmE9iUUkqDQq0UncCmlFIaFGqkxIaTr6uvKaUCnAYFt9TYCPJLKjHG+LsoSinlNxoU3FJjw6l2GorKqv1dFKWU8hsNCm46gU0ppTQo1Dq0LKf2KyilApcGBbfUOJsUT4elKqUCmQYFN20+UkopDQq1YsJDiAoL1kypSqmApkGhjtTYcPJLNCgopQKXBoU6UmLDydMJbEqpAOZRUBCRHiIS7n48VkRuFZF43xbt+EuNjdCOZqVUQPO0pjAHcIpIT2AWkAG80tRBIjJRRDaIyGYRuaue128Ske9FZKWILBKRfl6VvoWlaFI8pVSA8zQouIwxDmAq8IQx5k6gc2MHiEgw8BRwNtAPmF7PTf8VY8xAY8wQ4C/Ao16VvoWlxoVzsNJBeZXTn8VQSim/8TQoVIvIdOAnwHvubaFNHDMc2GyM2WKMqQJmA1Pq7mCMOVDnaTTg18RDKbrYjlIqwHkaFK4BRgAPGGO2ikh34L9NHJMG7KjzPNe97TAicrOI/IitKdxa34lE5EYRWSYiy/Lz8z0ssvd0AptSKtB5FBSMMWuNMbcaY14VkQQg1hjzcEsUwBjzlDGmB/Br4HcN7DPLGJNtjMlOSUlpibetV6pOYFNKBThPRx99LiJxIpIIrACeEZGm2v93Yjuka6S7tzVkNnC+J+XxldpZzTosVSkVoDxtPurgbv+/AHjRGHMycEYTxywFeolIdxEJAy4F5tXdQUR61Xl6DrDJw/L4RGJUGCFBohPYlFIBK8TT/USkM3Ax8FtPDjDGOETkFuBDIBh4zhizRkTuA5YZY+YBt4jIGUA1sB/bke03QUFCcky4prpQSgUsT4PCfdib+1fGmKUicgIefKs3xswH5h+x7Q91Ht/mRVmPC12rWSkVyDwKCsaYN4A36jzfAlzoq0L5U2psOLuKtU9BKRWYPO1oTheRuSKS5/6ZIyLpvi6cP6TG6axmpVTg8rSj+XlsJ3EX98+77m3tTkpMOIWllTicLn8XRSmljjtPg0KKMeZ5Y4zD/fMC4LsJA36UEheBMVBYWuXvoiil1HHnaVAoFJErRCTY/XMFUOjLgvlLzQQ2bUJSSgUiT4PCtdjhqHuA3cA04GoflcmvDi3LqZ3NSqnA42mai23GmMnGmBRjTKox5nza8egjQOcqKHWs8taDs9rfpVBeOpaV125vsVK0Iima/0ipY/fdS/CPk+F/Ry2jolq5YwkK0mKlaEXCQ4LpnhzNt1vaZZeJUr637j2Y93MIj4Nlz9sag2ozjiUo+HXtA186f0ga32wpZGdRub+LolTbsvVLePNaSBsGM76CsBj4qN7kx6qVajQoiMhBETlQz89B7HyFdumCrDSMgbkrcv1dFKXajl3fwauXQeIJcNnrEN8VxtwJmz+GzZ/4u3TKQ40GBWNMrDEmrp6fWGOMp3mT2pyMxCiGd0/krRU7MabdVoiUajkFm+ClCyEqAa58C6IS7fbhN0JCd/jwd+B0+LeMyiPH0nzUrk3LSmdLQSnf7Sjyd1GUat2Kc+HF80GC4Mq3Ia5OI0JIOJx5H+Svg+9e9F8Zlcc0KDTg7IGdiAgNYs5ybUJSqkGlhfDfqVB5AK6YA0k9jt6n73nQbSR89gBUHDj6ddWqaFBoQGxEKGf178S7q3ZR6XD6uzhKecdRBe/eBjMH2eGhLh/8DVcehJenQdF2mD4bOg+ufz8ROOsBKCuAhX9r+XKoFqVBoREXZqVzoMLBp+vy/F0UpTxXvh9evhCWvwDBofDOzfDP0bDpE2ipPjJHJcy+HHavgotegMyRje/fZSgMng7f/gP257RMGZRPaFBoxMieyXSMC9cmJNV27NsK/54A276B8/8JtyyzN+3qMhsoXpxib+THwuWEOdfD1i9gylPQ+2zPjhv/e5Bg+OSPx/b+yqc0KDQiOEg4f2gan2/Mp0DXbVat3Y6l8OwZUJIHV70NQ6bbppv+U+HmJTDxYdjzPfzrNHjrRtvs4y1j4L1fwLp5cNaD9j081SENRt4Ga96C7Yu9f291XGhQaMK0rHScLsM7K3f5uyhKNWzNXPjPuRAeA9d/ApmjDn89JAxOuQlu/Q5G/RLWvgNPZMNHv7fNTfWproA9P8APc2DBn+H1q+Cp4bDiRTjtThjxM+/LOfJWiOkEH/4GXLpmSWskbW0cfnZ2tlm2bNlxfc/JTy7C4TTMv230cX1fpZpkDCz6O3z6R8g4GS59BaKTmz6uONeOBlr1KkTGw6jbISYV8tdD/gb7s38rmJobt0Bid0juDSeMhZN/amshzfHdy/DOz+DCf8PAac07h/KaiCw3xmQ3uV/ABIWVr8DXT8JPv4Rg7+bdvfDVVu59dy0f3Daavp3jvH9vpXzBWQ3v/RK++y8MuBCm/ANCI7w7x+7V8PEfYMsC+zwoBJJ6QvKJkNIHUnrbn6SeEBrZMuV2ueCZsXY468+Xtdx5VaM8DQrtdlbyUcJiIG8NbPsKThjj1aGTh6TxwPx1vLUil9+e089HBVTKC+VFtjln6xe2KWfs3RDUjNbgzoNs/8Pu1XaiWeIJdsSSLwUFwVl/hhfOgW+egtPu8O37Ka8ETp9CzzMgNMq2pXopMTqMcb1TmfvdLl27Wfnf/hz3CKOvbe1g/O+aFxDq6jzI1gh8HRBqZI6CPufapq+De4/PeyqPBE5QCIuCXmfC+vea1cF14bB0CkoqWbipwAeFU6oJTgf8uMCmpP7naVCyx+YYGnq5v0vWfGfeZ+c7LLjf3yVpebnLoarM36VolsAJCgB9J0PJXtjh/XC4cb1TSYgKZY5mTlXHi8tlawPv/x882gf+ez788BaceBZc/yl0P83fJTw2ST1swrwV/7VDZduLH96CZ8fDSxdAZYm/S+O1wOlTAPufKTjcNiF1G+HVoWEhQUwe3IVXl+6guLyaDpHHqZqtAosxsHOFHQa6Zi4c3AUhkfZvd8AF0GtC++qYHXMnrHoF5t0K5z1mm7Hasv3b4N1f2L6ZHYvh1Uvh8jfa1GcWWDWF8FjoMR7Wvdus6f4XDkunyuHi/dW7fVA4FdAO7IZP7oXHBttvmUtmQZchdtjmnZvh4v9Avylt6ubikcgEmPSIHQL7r9Hwwrmwfn7bnMPgdNiZ3hi4ci5M/RfkLLLpQKor/F06jwVWTQHsf6yNH9hvY+nDvDp0YFoHeqXG8NaKXC47uauPCqgCTv4Gm2n04B47B2DMr6HPOXb+QCAYOA16nm4nxS2eBbOn22/aJ8+AIZfZCXltwecPQu4SG8gTMu2PoxLm3QJv/AQu/q+dRNjKBVZNAaD3RDsWe533o5BEhAuy0lm2bT85BaU+KJwKOLnL4Lmz7JyDGz8/1HkcKAGhRmSCTYFx20qY9hxEJsIHd8Kj/exynkU7/F3Cxm390maAHXrF4RPysq6Ecx6Fjf+DOdfaz7mV82lQEJGJIrJBRDaLyF31vH67iKwVkdUi8qmIdPNleQD7x9d9jO1XaEYT0tShaQQJvKUdzupYbf4U/jMZIjrAdR+2/fb0lhAcaifi3fApXPcJ9BwP3/zDNqu9cTXsWGJTdldX+CYdeHOUFtpcUkk94ey/HP36SdfZvFPr3oW5P2095W6Az5qPRCQYeAo4E8gFlorIPGPM2jq7fQdkG2PKRGQG8BfgEl+VqVa/KfDurXbEg5f/ETt1iGBkz2TmrNjJL844kaCgZk71V4Ht+zdh7k121vAVcyC2o79L1PpknAQZL9hawpJ/wfIXbef7YcQGkuAw2wIQHApBoTZrQfpJMHWW1xkMvGKMTU1eVgiXvQZh0fXvd8pN4Ky0s8eDw+z8kmOdW+IjvuxTGA5sNsZsARCR2cAUoDYoGGMW1Nn/W+AKH5bnkD7nHMr02IxvZxdmpfOL11ayJGcfp5yQ5IMCqnZtyTMw/07oOgKmvxp4TUXeis+ACffDmLvs/9myQtsM46wGV81vBzirDm0rL7IjuLoMhVN/7ruyLXnG9lFOfKjhRYZqjLzNLn604H4bvM59rFUGBl8GhTSgbkNgLnByI/tfB3xQ3wsiciNwI0DXri3QwRudbJcHXPuOnQ3qpbP6dyImPIQ5y3M1KLQXFcV2vHxsJzhxom86N42xnZFfPAy9J9m28/Y2msiXwmNsx7MnjIFXp9vsrn3Ps52+LW3P97a/o9dZcPJNnh0z5k5bY/jyr7bGMOmR5icW9JFWEaZE5AogG/hrfa8bY2YZY7KNMdkpKSkt86b9pkDBRshb7/WhkWHBTBrYifnf76asytEy5VH+4XLZYPDEMPjotzDnOvhrD3jtSjsJqaqFBhS4nHYS2hcPw5Ar7EgUDQi+IwLnPAISZJMGtnTiz6pSePNaW8s7/x/e3djH/RZOvRWWPgsf/rbxshlj+1CKd8Letbb/wsd8WVPYCWTUeZ7u3nYYETkD+C0wxhhz/Fay6XOurcKvmwepfbw+/MKsdF5flstHa/Zy/tA0HxSwBf0wB5Y8C5f817O0yoFixxL44Few6ztIHw7TX7Pf4tbMtbXIdfMOTRzrP9VOHAuL8v59HJW2g3HNXHszOPO+VvftsF3qkA6n/8F+xt+/AYMubrlz/+83ULDJJhP09v+UiP0bcFbDt0/ZXFbhsVB5ACoO2FprZbF9XHmgTvpy4Ny/Q/a1LXcd9RXPV6mzRSQE2Aicjg0GS4HLjDFr6uwzFHgTmGiM2eTJeVt0PYV/n2Uj/oxFXh/qchlO++sCuidH89/rGmsV87Py/fB4FpTvg8zRcOXbvu14awsO7IZP7oHVr0FsZzjjj/aGUfdG7XLC9m8OBYjSfJtQ8cSJNkD0GAeh0U23CVcehNeugC2f2xvByNt8emnqCC6nTR64fyvcvBSiW6C5d81cOxJq1O1wxj3NP48xtvlp5Su2aSy8gx2JFhEH4XH2d0SHwx93GdrsprBWsZ6CiEwCZgLBwHPGmAdE5D5gmTFmnoh8AgwEaqYIbzfGTG7snC0aFL55Cj68G36+wuZh8dKjH2/kic828fVd4+ncoZU2BfzvbrtY+shb4avHYMQtcNYD/i6Vf1RX2G9mX/7Ndkae+nP7H7up/gOX06ZcXzMX1s6DsjpJEYNCbOqUmhEwITWP3b/L98OBXTD5ibadvK4t27vGLkE68CKY+s9jO9f+bfDP0ZDcC6793/HLKtsCWkVQ8IUWDQpFO2DmADjjXrtEoZe2F5Zx2l8XcOdZvbl5XM+WKVNLKtgM/zgZhlwOkx+3zWVLZgXeilfGwPr3bZ/B/hzbdDjhfruSmLecDti2yM6Id1bZpiFn1aEfR9Xhz40LTroBTpzQ4pelvPDpn2DhI7am3GNc887hdMDzZ9vV6W5a6JvOax/SoOCpWe4/kBsXNL5fAy751zfsPVDBgjvGIq2tnfiVSyDnK7h1hV1q0VltJ0vt+g6u/xg6DWz+uZ3V9ltTh7TW3WFasMkGwy0L7JyAiQ81/6ag2q7qCnj6VDBOmPGN931DdVe5a6NfqjwNCq1i9JFf9ZsMu1ZA0fZmHX5RdgY5hWUs29bA4uf+svlTO7X+tDtsQABb1b3oBTtiYvblULaveec+sMumZnhyGDzQGf4+0Obumf8rO277x89sLcyfSc0cVfDFX+2NYNcKO6P0pkUaEAJVaITNwro/B754yLtjSwvgxfPpSHiLAAAaoUlEQVRtQBh9R5sMCN4I8B5H7BoLn9xrp6CPuNnrwycN7MQ97/zAm8tyOSkzseXL1xxOhx3qlpAJp8w4/LXYjnY45AuT7PDLy9+EoGDPz719Mbx+pe2gn/CA/V24yX4j3/EyVNXJHx8Saaf+J/WAqEQIiTj0Expx+POQcFvjiO0EnQYd2+ic3GV2MZq8tdD/Ajj74UOBUQWu7qNh6JV2rfYBFzY92Qxgzw92vkNpHlzwTMuOYGqlNCgk9YCOA20HYjOCQlRYCJMGdua91bu4Z3I/osJawT/p8uchfx1c8pK92R4p4yQ7aebdW+Gz+z0fQbH8P3asfYd0uOodSO17+OvG2EWMCja5A8Vm+3vP93ZoXXUFOCpsJ29jOg+xwaz/1PrL35DKEvjsT7D4XxDXBabPht5ne368av8m/Ak2fmjXb7jhs8a/EK19x6YiiYiHaz6AtKzjV04/agV3sFag32RY8IAdqhjX2evDL8rO4I3luXzw/R4uHJbugwJ6oXy/ncWZOdp2qDZk2E9ss8qiR+03pv7nN7yvsxr+d5edbNNjvG1TjaqnViRiv+nHdrLfyhrictrg4KiE6vJDjx3ltr9j8b/suP6Pfm/HZGdf23RuoE0f2zbf4lwYfgOM/70dxqdUXZEJtub45jWw+J/1fxF0uewkwy8esvmTLnnJ/k0HCO1TANuEBHb95mY4KTOBzKQo3ljeQHrfigPw9RN2RqKvffFXGxgmPth0E8zZf7F/9G//DPLW1b9PST68OMUGhFN/Dpe9UX9A8EZQsE0cFpVoO6qTekDHfpA2DE66Hm5eAle8Zcdkf/EQ/L2/zUK5c8XR5yotsAubvDzNziO49kOY9FcNCKph/afa1BSf3W8HS9RVWWKbR794yI7au/r9gAoIoEHBSu0DySfa6mIziAjThqXz7ZZ9bC+ss1i302Fvpo8PtZNUZk+3k5l8pWCzzSaZdZVnI4tCwm3/Qli07XguLzr89V0rYdZY2LnctqdOuP/4THwTsYuuXP66nUNy0nV2Na5nxsGzZ9oZ2s5qWDUbnjwJ1rwNY39jhwl2bcUTCVXrIALn/M2mwHj/9kNpJvbn2IluG+bDWQ/ClKe8a75sJzQo1Og3xU5QKi1oet96XJCVjgjMWZFr/8g2/A+eHmHb4JNPtNPT92+zHcC+8tFvbeeuN0n+4jrDxS9C0TZ3rnf3iKHVb9gRRmC/ffurgy2ph63u377WjiAqK7A5Zx7ubsub3MuOKhp7V0D+B1bNFJ9hmxg3f2K/ZGz90g5PP7DTpjIf8bOATUWifQo1+k62mQvXvwfDrvb68C7xkYzqmcyqpV9idv4fkvMlJPaAS162qbpFbFD4aqbt/GzpDtCaIahn3uf9SJtuI+z4/fl3wOd/tu37Xz9uM8le9B+IaaEkhMciIs7mpB9+I2z+GFa/bss97NpWmX5YtQHDb4DvX4f3brej5pJ62lTmzchu0J7o5LUaxsDjQ+yN/Mq3vD++eCfb37yb9O3v4IyIJ3T8b2wHad1p8I5KeOZ0KNkDP/u25ZLTOR3wz1G2o/bmJc37xlyzWMjKl+3zk663gaINTeNXymt7foBnT7drY1/wTLvui/J08prWFGqI2Cakb56yHbWRCZ4dV3nQ5hT6+kkyjJPn5Tw2dbuRB0+uZ/RNSDhcMAtmjYF3b7OjGlqiitrUEFRPiNi1ZAG6nWrXmlWqves0AO7YaJPOBWhz0ZG03l1X3yl2BacN9a71c4jLBTuWwqf32QykX/4V+kxCblnK5sG/Yu66Eg5WNDAWv2M/m853/XuHvpUfC0+HoHoiNMLmhteAoAJJRAcNCHVoTaGutCyIS7cT2Y5c4amq1KY/3vCBnfxSmgcSDN1Pg/GvQrqtlV00bD+vLN7O+6t3c+nwBlaJO+Vme44Pfg2Zo44tsZY3Q1CVUqoJGhTqErFL9y17zjYLVZbYztsNH8DWL+wkq/A46HmG7SjuecZRY/aHZMTTMzWGN5bnNhwUgoLsN/KnR8LcGXD1e96lmqhRsMm7IahKKdUEDQpH6jcFFj9tb9hF7okt8d1g2DXQeyJ0PRVCwho8XES4aFg6D36wnh/zS+iR0kCu/viudvLY2zfZiW2jfuFdOStLbPZPb4egKqVUIzQoHCnjZPtjXDYVRO9JNuWyF00zU4em8ZcPNzBneS6/mtjIUp+DL7UTZT67307W8uTbvjF2acGP/wAHd9scRprsTSnVQjQoHCkoCK776JhOkRoXwZgTU3hrxU7+b0JvgoMaCCgicO5M2P4tvPVTm6ArNKLhE+/6zvZD7Fhsk8Zd9B+dwauUalE6+shHLhqWzp4DFSzclN/4jtFJdjp93hpYcH/9+5Tk21TQs8bBvi0w+Um4YYEGBKVUi9Og4COn9+1IQlQobyzPbXrnEyfYiW5fPwk5iw5td1bbeRNPDLOLe4+4GX6+HLKu1Fm8Simf0DuLj4SFBDFlSBofr9lLUVlV0wfUrBk89yaoKLY5WZ4+FT68265/MOMbOOsBO6ZaKaV8RIOCD00blk6V08W7q3Y1vXNYNEydZRNy/WMEvHShnUg3/TW7OlrKib4vsFIq4GlQ8KEBaR3o2znOsyYksDWCcXfb9RfO+KPNj9R7ok5KU0odNxoUfOyiYemszi1mwx4P11E47U64a7udt6CpoJVSx5kGBR87f2gaocHCG8saWJWtPtqJrJTyE737+FhidBjj+6Ty9sqdVDtd/i6OUko1SoPCcXDRsAwKSqp4+dttOF1ta/0KpVRg0aBwHIztnULvjrHc++5aRj/8GTM/2cju4nJ/F0sppY6iK68dJ9VOF5+s3csrS7azcFMBQQLj+3TkspMzGHNiasOpMJRSqgXoymutTGhwEGcP7MzZAzuzvbCM2Uu38/qyXD5Zt5e0+EguOSmDi7Mz6NShkdxHSinlYz6tKYjIROAxIBh41hjz0BGvnwbMBAYBlxpj3mzqnG21plCfKoeLT9bt5VV37SE4SBjfJ5XLTu7KmF4pBGntQSnVQvxeUxCRYOAp4EwgF1gqIvOMMWvr7LYduBq4w1flaM3CQoKYNLAzkwZ2ZlthKbOX7uCNZTv4eO1eeqXGcMNpJzBlSBfCQ5qxAI9SSjWDLzuahwObjTFbjDFVwGxgSt0djDE5xpjVQMCP1eyWFM2vJ/bh67tO5++XDCY4SPjVm6sZ/fACnv78R4rLG1jzuQlbC0p5duEW7nt3LV9tLsChw2KVUo3wZZ9CGlB3xlYu0KxczyJyI3AjQNeuDSxx2U6EhQQxdWg65w9JY+GmAmZ9uYWH/7eeJz/bxPThXbl2VHe6xEc2eHyVw8WynH18uj6Pz9bnsbWg1J43OIjnvtpKckwYEwd04pyBXRjePVE7uJVSh2kTHc3GmFnALLB9Cn4uznEhIpx2YgqnnZjCDzuLeWbhFp7/OocXvs7hvMFduGH0CfTrEgdAYUkln2/I57P1eXy5MZ+DlQ7CgoMY0SOJa0ZmMq53Kimx4Xy+IY/3Vu9mzvKdvPTtdpJjwpk0sBPnDOxMdqYGCKWUDzuaRWQEcK8x5iz3898AGGMerGffF4D3Aq2j2Vs7i8p5btFWZi/ZTmmVk1N7JFFe7WTljiKMgdTYcMb3SWV8n1RG9kwmOrz+mF9W5WDB+nze/34Xn63Po6LaRWpsOJMGduacQZ3J7paAaBI+pdoVTzuafRkUQoCNwOnATmApcJkxZk09+76ABgWPFZdV8/KSbbyyeDtJ0WGM79OR8X1S6d8lzusRS6WVDj5bn8f7q3ezYEMelQ4XY3un8NilQ+kQGeqjK1BKHW9+DwruQkzCDjkNBp4zxjwgIvcBy4wx80TkJGAukABUAHuMMf0bO6cGBd8pqXTw2tIdPPTBOtIToph15TB6dYw9pnMu3lLIPfPWcO6gztwyvlcLlVQp5a1WERR8QYOC7y3N2ceMl1ZQUe3k75cM4cx+Hb0+R6XDyaMfbWTWwi1EhQZTWuXk1tN78cszemnTlFJ+4GlQ0NxH6ignZSby7s9HckJKNDe8uIzHP92Ey4tEfuv3HGDKk1/xry+3cOlJXfn27tO5JDuDxz/dxKMfb6StfRFRKpC0idFH6vjr3CGS1386grvf+p5HP97I2l0HeOTiwcQ00HkN4HIZnvtqK3/53wbiIkN49qpsznDXMh68YCAi8MRnm3G6DHee1VtrDEq1QhoUVIMiQoP528WD6Z/WgT/PX8cF//iKZ67KpltS9FH77iwq547XV/HNlkLO6NuRhy4cSHLMoZXjgoKEP08dSFCQ8I/Pf8Rl4NcTNTAo1dpoUFCNEhGuG9WdPp1iufmVFUx+8iuevGwoo3ulAGCM4Z2Vu/j9Oz/gdBkevnAgF2dn1HuzDwoS7p8ygCCBf37xI8YY7jq7jwYGpVoR7VNQHhnZM5l5N4+ic4cIfvLcEmZ9+SP7S6u45dXv+MVrKzmxYywf3DaaS07q2uhNPihI+NOUAVw1ohv/+nILf56/TvsYlGpFtKagPNY1KYo5M07lzjdX8ef565n5ySaqHC7uPKs3N43p4fGMaBHhj5P7EyTCMwu34jLwu3P6elRjOFBRzcKNBSzaXEDHuHBG90pmUHo8ocH6/UaplqBBQXklOjyEpy7L4ukvfuTz9fn84bx+DEjr4PV5RIR7zuuHCPx70VZcxvCHc/vVGxi2FZbyybo8Plu/l8Vb9uFwGWLDQyipcjDzk03EhIdwyglJjOqZxKheKfRIifa4SWpfaRWb9h5kY14JOQWl9EyN4Yy+HUmJDW/6YKXaIZ2noPzKGMP976/j34u28pMR3bh3cn+cLsOK7UV8um4vn6zby4/5Nqlfr9QYTu/bkdP7ppLVNYGDFdV8/WMhizYXsGhTAdv3lQHQuUMEI3smM7pXMqf2SCYlNpzCkko25ZWwae9BNuWVsHHvQTbnlVBQUlVblrDgIKqcLkRgWNcEJvTvyIR+nchMPrpjXam2RievqTbDGMOf56/jmYVbyeoaz5aCUorKqgkNFk45IYnxfVI5vU9HuiZFNXqe7YVlNkBszuerzYW16cY7RIYelno8JjyEnqkxnNgxhl6psfTqGEOvjrF0jotgw96DfLRmLx+t3cOaXQcAOLFjDBP6dWJC/44MTOugHeOqTdKgoNoUYwx//3gjc1fuZHhmEqf3TWV0r2RiI5qXf8npMqzZVczCTQXk7i+nR0o0vTrGcmLHGDrFRXh0Y8/dX8bHa/fy0Zq9LMnZh9Nl6Nwhggn9OjKmdwo9U2JJS4jU7LKqTdCgoFQL2l9axafr8/hozR6+3JRPRbVdrCg0WOiaGEX35Gi6J0eT6f7dPTmajrERuqSqajX8vhynUu1JQnQY04alM21YOuVVTr7fWUxOQSlbC0vZml9KTmEpCzcVUOk4tLJdRGgQmUnR9OsSx5CMeAanx9Onc6wur6paNQ0KSnkpMiyY4d0TGd498bDtLpdhz4EKthaU1v5syS/hy40FvLViJ2A7s/t2jmWwO0gMzojnhORorVG0IJfLsPdgBWDTtSjvaPORUj5mjGF3cQWrdhSxMreIVTuK+D63mNIqJwCx4SEMyuhAdrdEzhvcmZ6px5auvK0wxrDNPThgx74y4iJD6RAZSnyU+3dkGB0iQ+kQFUpseEht4DTGsK+0ih37y9mxr4wd+8vIdT/O3V/Ozv3lVLnXIu+ZGsO43imM7Z3KSZmJhIUE7nwW7VNQqhVzugxb8ktYuaOIVblFrNpRzJpdxbgM9O8Sx/lD0pg8pAsd4yL8XdQWVVhSydc/FvLV5gIWbipgZ1E5YPtmqp0N34uCBOIiQ4mNCKGwpIoyd0CtkRAVSkZiFBkJUaQnRpKREEVFtZMvNuazeMs+qpwuosOCGdkzmXF9UhnbO8WjWoTLZSgorWTn/nJ2FVUgAiN7JNMhqu0tQKVBQak2Ju9gBe+t2s07K3eyKrcYERhxQhLnD0lj4sBOxDVzJFZLqXK4WLQ5n017Sw59q48MPewbfkx4yGEju8qrnCzJ2cdX7rkka3fbYb6xESGc2iOJUT2TGdkzme7J0VQ6XBwor6aovJri8mqKyqopKquiuM7zAxXVJEaHkZEQRUZiFOkJkWQkRjWavbe00sE3PxayYEMen2/Irw1EfTrFMrZ3Kqf1SsZlYFdROblF5exy/+wsKmd3UUVtraNGcJAwrGsCY/ukML5PKr07xraJYcoaFJRqw7bkl/DOyl28s3InOYVlhIUEcUbfVKYMSWNs75TDOqsdThelVU7KqhyUVtrfZe7nocFBDEqLb/Y325pA8P7qPXy8dg8HKhyN7h8cJMRFhBAfFUZkaDCb80qocroIDRaGdUtgVM9kRvVKYUCXOEL8kJrEGMOmvBIWrLcBYmmOnSFfQwQ6xkbQJT6CLvGRpCVEkhYfSZcO9nHN+uafrc+rDXBdOkQwrk8q43qncmrPJKLCWmdXrQYFpdoBYwyrcot5+7udvLd6FwUlVcRGhBAXEWqDQJWTKoeryfP0TI0hq2s8WV0TyOqWQM+UmAY7t+sGgo/W7uFghYPYiBAm9OvEOYM6kZ2ZSGmlo/bbe3F5NcXu30XlVbXbD1Y4OLFjDKN6pXBSZkKrvFkerKhmWc5+IsOCSYuPpGNchMf9DnuKK/h8Qx6frc9j0eYCyqqchIUEMeKEJMb1TuHUnsmN/jsfbxoUlGpnHE4XizYX8OGaPVQ6XESHhRAVHmx/hwUTHe7+XWd7SaWD77bvZ8X2IlZs309RmZ3ZHRsRwpAMGySGdUugf5c4VuUW1RsIzh3UmZE9kwO6k7YplQ4nS7fu57P1eXy+IY8tBTY1S0JUKNmZiQzPTOSk7on07xLnt+SNGhSUUocxxrCloJQV22yQ+G77fjbsPUjdW4AGgpaxrbCUxVv3sXTrPpbm7COn0ObligoLZmjXeIZnJnFS9wSGZiQQGXb4vBVjDJUOF5XVLsqrnVRUO6lwOCmvcpKWEElqbPMGH2hQUEo16WBFNat2FPPDrmJ6d4zVQOAjeQcqWJqznyVbC1mSs5/1ew5gjB11lRYfSaXDZW/+1S4qHE4aui3ff/4ArjilW7PKoEFBKaVaqeLyalZs28+SnH3k7i8nIiSIyLBgIkKDiQgJIjzUPo4MDSYiNMhuDw2id6c40uKbNyFP01wopVQr1SEy1I5Y6pPq76IcReuJSimlamlQUEopVUuDglJKqVoaFJRSStXSoKCUUqqWBgWllFK1NCgopZSqpUFBKaVUrTY3o1lE8oFtzTw8GShoweK0Bu3tmtrb9UD7u6b2dj3Q/q6pvuvpZoxJaerANhcUjoWILPNkmndb0t6uqb1dD7S/a2pv1wPt75qO5Xq0+UgppVQtDQpKKaVqBVpQmOXvAvhAe7um9nY90P6uqb1dD7S/a2r29QRUn4JSSqnGBVpNQSmlVCM0KCillKoVMEFBRCaKyAYR2Swid/m7PMdKRHJE5HsRWSkibXIpOhF5TkTyROSHOtsSReRjEdnk/p3gzzJ6o4HruVdEdro/p5UiMsmfZfSWiGSIyAIRWSsia0TkNvf2Nvk5NXI9bfZzEpEIEVkiIqvc1/RH9/buIrLYfc97TUTCPDpfIPQpiEgwsBE4E8gFlgLTjTFr/VqwYyAiOUC2MabNTrgRkdOAEuBFY8wA97a/APuMMQ+5g3eCMebX/iynpxq4nnuBEmPMI/4sW3OJSGegszFmhYjEAsuB84GraYOfUyPXczFt9HMSEQGijTElIhIKLAJuA24H3jLGzBaRfwKrjDFPN3W+QKkpDAc2G2O2GGOqgNnAFD+XKeAZY74E9h2xeQrwH/fj/2D/w7YJDVxPm2aM2W2MWeF+fBBYB6TRRj+nRq6nzTJWiftpqPvHAOOBN93bPf6MAiUopAE76jzPpY3/IWA/9I9EZLmI3OjvwrSgjsaY3e7He4CO/ixMC7lFRFa7m5faRDNLfUQkExgKLKYdfE5HXA+04c9JRIJFZCWQB3wM/AgUGWMc7l08vucFSlBoj0YZY7KAs4Gb3U0X7YqxbZttvX3zaaAHMATYDfzNv8VpHhGJAeYAvzDGHKj7Wlv8nOq5njb9ORljnMaYIUA6tmWkT3PPFShBYSeQUed5untbm2WM2en+nQfMxf4htAd73e2+Ne2/eX4uzzExxux1/4d1Ac/QBj8ndzv1HOBlY8xb7s1t9nOq73raw+cEYIwpAhYAI4B4EQlxv+TxPS9QgsJSoJe7Nz4MuBSY5+cyNZuIRLs7yRCRaGAC8EPjR7UZ84CfuB//BHjHj2U5ZjU3TreptLHPyd2J+W9gnTHm0TovtcnPqaHracufk4ikiEi8+3EkdkDNOmxwmObezePPKCBGHwG4h5jNBIKB54wxD/i5SM0mIidgawcAIcArbfF6RORVYCw2ze9e4B7gbeB1oCs2RfrFxpg20XnbwPWMxTZJGCAH+GmdtvhWT0RGAQuB7wGXe/Pd2Hb4Nvc5NXI902mjn5OIDMJ2JAdjv+i/boy5z32fmA0kAt8BVxhjKps8X6AEBaWUUk0LlOYjpZRSHtCgoJRSqpYGBaWUUrU0KCillKqlQUEppVQtDQpKHUFEnHWyZa5syay6IpJZN4uqUq1NSNO7KBVwyt0pA5QKOFpTUMpD7jUs/uJex2KJiPR0b88Ukc/cydQ+FZGu7u0dRWSuO8/9KhE51X2qYBF5xp37/iP3LFSlWgUNCkodLfKI5qNL6rxWbIwZCDyJnSEP8ATwH2PMIOBl4HH39seBL4wxg4EsYI17ey/gKWNMf6AIuNDH16OUx3RGs1JHEJESY0xMPdtzgPHGmC3upGp7jDFJIlKAXbil2r19tzEmWUTygfS6qQXc6Zo/Nsb0cj//NRBqjLnf91emVNO0pqCUd0wDj71RN/+ME+3bU62IBgWlvHNJnd/fuB9/jc28C3A5NuEawKfADKhdBKXD8SqkUs2l31CUOlqkexWrGv8zxtQMS00QkdXYb/vT3dt+DjwvIncC+cA17u23AbNE5DpsjWAGdgEXpVot7VNQykPuPoVsY0yBv8uilK9o85FSSqlaWlNQSilVS2sKSimlamlQUEopVUuDglJKqVoaFJRSStXSoKCUUqrW/wNkv2ZhG6hyGAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluation = model.evaluate(x = X_test, y = y_test, verbose=1) \n",
    "loss = evaluation[0]\n",
    "acc = evaluation[1]\n",
    "print('LOSS ON TEST DATA='+str(loss))\n",
    "print('ACCURACY ON TEST DATA='+str(acc))\n",
    "\n",
    "\n",
    "# Plot training & validation loss values\n",
    "plt.plot(history.history['loss'], label='Train')\n",
    "plt.plot(history.history['val_loss'], label='Validation')\n",
    "plt.title('Model loss')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Validation'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.99      0.98       491\n",
      "           1       0.94      0.88      0.91        88\n",
      "\n",
      "   micro avg       0.97      0.97      0.97       579\n",
      "   macro avg       0.96      0.93      0.94       579\n",
      "weighted avg       0.97      0.97      0.97       579\n",
      "\n",
      "Accuracy: 0.9723661485319517\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAF4ZJREFUeJzt3X20XXV95/H3xwREBYMasJoEgjxUqU9oCjiupcyIFBkNLusDGdHiUNJaaXW0ts6qo5RWrTrasQWrVCiKCoJjXWmNxSc0oxVNHJACCo3IQwIOIBBEMDx954+9Lzlekn1Prnffc254v9a6i7P3+Z29v/vHzfncvX/n/HaqCkmStuVhoy5AkjTeDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0JzSpLLkhw2RZu9ktyRZN4sldW7JFcnObx9fFKST466Jj10GBSaEe0b2V3tG/T/S3Jmkl1nej9V9RtV9fUp2lxbVbtW1X0zvf/2Tfqe9jhvS/KvSZ4z0/uRxolBoZn0kqraFXgWsAx4++QGacz137vPtMe5ELgAOG/E9cy4JPNHXYPGx1z/B6sxVFUbgS8CTwVI8vUk70ryLeBO4ElJFiQ5PckNSTYm+cvBS0VJTkjygyQ/S3J5kme16wcvwRycZF2S29uzmA+265cmqYk3uyRPTLIqyS1J1ic5YWA/JyU5N8kn2n1dlmTZkMd5L/ApYFGSPQa2+eIkFw+ccTx94LklST6X5KYkP01ySrt+3yRfa9fdnORTSXafTv8nObrd/+1JfpTkyMl9N3Dsn5zUZ8cnuRb4WpIvJjlx0ra/n+Rl7eMnJ/ly269XJHnldOrV+DMoNOOSLAGOAi4aWP0aYCWwG3ANcCZwL7AfcBBwBPC77etfAZwEvBZ4NLAc+OlWdvUh4ENV9WhgX+DcbZR0DrABeCLwcuDdSf7TwPPL2za7A6uAU4Y8zp3bGn8K3NquOwg4A/g94HHAR4FVSR7eBuE/t8e/FFjU7hcgwHvaGp8CLGn7YLskORj4BPDW9nieB1y9HZt4frv/3wLOBlYMbPtAYG/gC0keBXwZ+DSwJ3AM8OG2jXYwBoVm0ueT3AZ8E/gG8O6B586sqsvav8IfSxMkb6qqn1fVjcBf07zZQBMY76uqtdVYX1XXbGV/9wD7JVlYVXdU1YWTG7Sh9VzgT6vqF1V1MfAxmjf4Cd+sqtXtmMZZwDOmOM5Xtsd5F3AC8PL2uKAJw49W1Xeq6r6q+jiwGTgUOJgmCN7aHvcvquqbAO0xfrmqNlfVTcAHad60t9fxwBnttu6vqo1V9cPteP1JbW13Af8IPDPJ3u1zrwY+V1WbgRcDV1fVP1TVvVV1EfC/gVdMo2aNOYNCM+mlVbV7Ve1dVX/QvtlMuG7g8d7ATsAN7eWZ22j+8t6zfX4J8KMh9nc8cADwwyRrk7x4K22eCNxSVT8bWHcNzV/zE34y8PhOYJck85O8uh20viPJFwfanFtVuwOPBy4Fnj3p2N4ycVztsS1p61gCXDMQKg9I8vgk57SX4W4HPkkzBrK9hu27bXng/1PbZ19gS4CvoLnUBs1xHjLpOF8N/NqvsG+NKQesNFsGpym+juav7IVbe9Nsn993yg1W/Tuwoh0cfxnw2SSPm9TseuCxSXYbCIu9gI1DbP9TbHlj3NrzNydZCaxL8umquqGt/V1V9a7J7dtPR+2VZP5WjvvdNH30tKq6JclLGfIS2CRdffdz4JEDy1t7U588nfTZwDuTrAF2oRm8n9jPN6rqhdOoUXOMZxSade0b6peADyR5dJKHtYO5E5daPgb8cZJnNx+Syn4Dlz8ekOTYJHtU1f3Abe3q+yft6zrgX4H3JNmlHVg+nuYv9pk4liuA84E/aVf9PfD7SQ5pa39Ukv+cZDfgu8ANwF+163dJ8tz2dbsBdwCbkiyiGWOYjtOB1yV5Qduvi5I8uX3uYuCYJDu1A/YvH2J7q2nOHk6m+bTXRP/+M3BAkte029spyW8meco069YYMyg0Kq8FdgYupxkI/izwBICqOg94F81A6c+Az9OMa0x2JHBZkjtoBraPmXS5a8IKmsHj62muu7+zqr4yg8fyfmBlkj2rah3NuMUp7XGtB44DaMdAXkIzgH8tzQD7q9pt/DnNx4o30Vzu+dx0Cqmq7wKvoxnz2UQzVjQRsv+D5mzj1nZ/nx5ie5vbWg4fbN+enR1Bc1nqeprLd+8FHj6dujXe4o2LJEldPKOQJHUyKCRJnQwKSVIng0KS1GnOfY9i4cKFtXTp0lGXIUlzyve+972bq2qPqVs+2JwLiqVLl7Ju3bpRlyFJc0qSrU2DMxQvPUmSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTr0FRZIzktyY5NJtPJ8kf5PmHsaXpL0nsiRpvPR5RnEmzTTQ2/IiYP/2ZyXwdz3WIkmapt6CoqrWALd0NDka+ER7T+QLgd2TPGGq7d5990xVKEkaxijHKBbxy/dR3sAv38f4AUlWJlmXZN0NN9w6K8VJkhpzYjC7qk6rqmVVtWzBgseMuhxJekgZZVBsBJYMLC9miBveS5Jm1yiDYhXw2vbTT4cCm6rqhhHWI0nait5mj01yNnAYsDDJBuCdwE4AVfURYDVwFM3N5++kuSG8JGnM9BYUVbViiucLeENf+5ckzYw5MZgtSRodg0KS1MmgkCR1MigkSZ0MCklSp94+9dSX+++HK68cdRWS1HjsY2HhwlFX0a85FxQAa9aMugJJgs2bm6BY0fllgLlvzgXF/Plw0EGjrkKS4Jpr4JauObJ3EI5RSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6tRrUCQ5MskVSdYnedtWnt8ryQVJLkpySZKj+qxHkrT9eguKJPOAU4EXAQcCK5IcOKnZ24Fzq+og4Bjgw33VI0manj7PKA4G1lfVVVV1N3AOcPSkNgU8un28ALi+x3okSdMwv8dtLwKuG1jeABwyqc1JwJeS/CHwKODwrW0oyUpgJcCee+4144VKkrZt1IPZK4Azq2oxcBRwVpIH1VRVp1XVsqpatmDBHrNepCQ9lPUZFBuBJQPLi9t1g44HzgWoqm8DuwALe6xJkrSd+gyKtcD+SfZJsjPNYPWqSW2uBV4AkOQpNEFxU481SZK2U29BUVX3AicC5wM/oPl002VJTk6yvG32FuCEJN8HzgaOq6rqqyZJ0vbrczCbqloNrJ607h0Djy8HnttnDZKkX82oB7MlSWPOoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJneYP2zDJImDvwddU1Zo+ipIkjY+hgiLJe4FXAZcD97WrC+gMiiRHAh8C5gEfq6q/2kqbVwIntdv7flX9l2GLlyT1b9gzipcCv15Vm4fdcJJ5wKnAC4ENwNokq6rq8oE2+wP/HXhuVd2aZM/hS5ckzYZhxyiuAnbazm0fDKyvqquq6m7gHODoSW1OAE6tqlsBqurG7dyHJKlnw55R3AlcnOSrwANnFVX1Rx2vWQRcN7C8AThkUpsDAJJ8i+by1ElV9S9D1iRJmgXDBsWq9qeP/e8PHAYsBtYkeVpV3TbYKMlKYCXAnnvu1UMZkqRtGSooqurjSXamPQMArqiqe6Z42UZgycDy4nbdoA3Ad9pt/TjJlTTBsXbS/k8DTgM44IBlNUzNkqSZMdQYRZLDgH+nGZz+MHBlkudN8bK1wP5J9mlD5hgefFbyeZqzCZIspAmiq4YtXpLUv2EvPX0AOKKqrgBIcgBwNvDsbb2gqu5NciJwPs34wxlVdVmSk4F1VbWqfe6IJBMfu31rVf10+ocjSZppwwbFThMhAVBVVyaZ8lNQVbUaWD1p3TsGHhfw5vZHkjSGhg2KdUk+BnyyXX41sK6fkiRJ42TYoHg98AZg4uOw/4dmrEKStIMb9lNPm4EPtj+SpIeQzqBIcm5VvTLJv9HMxfRLqurpvVUmSRoLU51RvLH974v7LkSSNJ46v0dRVTe0D28Grquqa4CHA88Aru+5NknSGBh2UsA1wC7tPSm+BLwGOLOvoiRJ42PYoEhV3Qm8DPhwVb0C+I3+ypIkjYuhgyLJc2i+P/GFdt28fkqSJI2TYYPiTTQ3GPrHdhqOJwEX9FeWJGlcDPs9im8A3xhYvootX76TJO3Apvoexf+qqjcl+Se2/j2K5b1VJkkaC1OdUZzV/vd/9l2IJGk8dQZFVX2vfbgOuKuq7gdIMo/m+xSSpB3csIPZXwUeObD8COArM1+OJGncDBsUu1TVHRML7eNHdrSXJO0ghg2Knyd51sRCkmcDd/VTkiRpnAx7P4o3AecluR4I8GvAq3qrSpI0Nob9HsXaJE8Gfr1ddUVV3dNfWZKkcTHUpackjwT+FHhjVV0KLE3i1OOS9BAw7BjFPwB3A89plzcCf9lLRZKksTJsUOxbVe8D7gFoZ5JNb1VJksbGsEFxd5JH0E7jkWRfYHNvVUmSxsawn3p6J/AvwJIknwKeCxzXV1GSpPExZVAkCfBDmpsWHUpzyemNVXVzz7VJksbAlEFRVZVkdVU9jS03LZIkPUQMO0bxf5P8Zq+VSJLG0rBjFIcAxya5Gvg5zeWnqqqn91WYJGk8DBsUv9VrFZKksTXVHe52AX4f2A/4N+D0qrp3NgqTJI2HqcYoPg4sowmJFwEf6L0iSdJYmerS04Htp51Icjrw3f5LkiSNk6nOKB6YIdZLTpL00DRVUDwjye3tz8+Ap088TnL7VBtPcmSSK5KsT/K2jna/naSSLNveA5Ak9avz0lNVzZvuhpPMA04FXghsANYmWVVVl09qtxvwRuA7092XJKk/w37hbjoOBtZX1VVVdTdwDnD0Vtr9BfBe4Bc91iJJmqY+g2IRcN3A8oZ23QPa+3AvqarOqUGSrEyyLsm6TZtumvlKJUnb1GdQdEryMOCDwFumaltVp1XVsqpatmDBHv0XJ0l6QJ9BsRFYMrC8uF03YTfgqcDX26lBDgVWOaAtSeOlz6BYC+yfZJ8kOwPHAKsmnqyqTVW1sKqWVtVS4EJgeVWt67EmSdJ26i0o2u9dnAicD/wAOLeqLktycpLlfe1XkjSzhp0UcFqqajWwetK6d2yj7WF91iJJmp6RDWZLkuYGg0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdeg2KJEcmuSLJ+iRv28rzb05yeZJLknw1yd591iNJ2n69BUWSecCpwIuAA4EVSQ6c1OwiYFlVPR34LPC+vuqRJE1Pn2cUBwPrq+qqqrobOAc4erBBVV1QVXe2ixcCi3usR5I0DX0GxSLguoHlDe26bTke+OLWnkiyMsm6JOs2bbppBkuUJE1lLAazkxwLLAPev7Xnq+q0qlpWVcsWLNhjdouTpIe4+T1ueyOwZGB5cbvulyQ5HPgz4PlVtbnHeiRJ09DnGcVaYP8k+yTZGTgGWDXYIMlBwEeB5VV1Y4+1SJKmqbegqKp7gROB84EfAOdW1WVJTk6yvG32fmBX4LwkFydZtY3NSZJGpM9LT1TVamD1pHXvGHh8eJ/7lyT96sZiMFuSNL4MCklSJ4NCktSp1zEKSdrRbd4MV1456ir6ZVBI0jQtWAA/+QmsWTPqSoax26Om+0qDQpKmaffd4ZBDRl3FsObNm+4rHaOQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVKnXoMiyZFJrkiyPsnbtvL8w5N8pn3+O0mW9lmPJGn79RYUSeYBpwIvAg4EViQ5cFKz44Fbq2o/4K+B9/ZVjyRpevo8ozgYWF9VV1XV3cA5wNGT2hwNfLx9/FngBUnSY02SpO00v8dtLwKuG1jeAByyrTZVdW+STcDjgJsHGyVZCaxsl+5ZtuwxV/dS8ZyzeQE8fNOoqxgP9sUW9sUW9sUWty+e7iv7DIoZU1WnAacBJFlXdeuyEZc0Fpq+uNO+wL4YZF9sYV9skWTddF/b56WnjcCSgeXF7bqttkkyH1gA/LTHmiRJ26nPoFgL7J9knyQ7A8cAqya1WQX8Tvv45cDXqqp6rEmStJ16u/TUjjmcCJwPzAPOqKrLkpwMrKuqVcDpwFlJ1gO30ITJVE7rq+Y5yL7Ywr7Ywr7Ywr7YYtp9Ef+AlyR18ZvZkqROBoUkqdPYBoXTf2wxRF+8OcnlSS5J8tUke4+iztkwVV8MtPvtJJVkh/1o5DB9keSV7e/GZUk+Pds1zpYh/o3sleSCJBe1/06OGkWdfUtyRpIbk1y6jeeT5G/afrokybOG2nBVjd0PzeD3j4AnATsD3wcOnNTmD4CPtI+PAT4z6rpH2Bf/EXhk+/j1D+W+aNvtBqwBLgSWjbruEf5e7A9cBDymXd5z1HWPsC9OA17fPj4QuHrUdffUF88DngVcuo3njwK+CAQ4FPjOMNsd1zMKp//YYsq+qKoLqurOdvFCmu+s7IiG+b0A+AuaecN+MZvFzbJh+uIE4NSquhWgqm6c5RpnyzB9UcCj28cLgOtnsb5ZU1VraD5Bui1HA5+oxoXA7kmeMNV2xzUotjb9x6Jttamqe4GJ6T92NMP0xaDjaf5i2BFN2RftqfSSqvrCbBY2AsP8XhwAHJDkW0kuTHLkrFU3u4bpi5OAY5NsAFYDfzg7pY2d7X0/AebIFB4aTpJjgWXA80ddyygkeRjwQeC4EZcyLubTXH46jOYsc02Sp1XVbSOtajRWAGdW1QeSPIfm+1tPrar7R13YXDCuZxRO/7HFMH1BksOBPwOWV9XmWapttk3VF7sBTwW+nuRqmmuwq3bQAe1hfi82AKuq6p6q+jFwJU1w7GiG6YvjgXMBqurbwC7AwlmpbrwM9X4y2bgGhdN/bDFlXyQ5CPgoTUjsqNehYYq+qKpNVbWwqpZW1VKa8ZrlVTXtydDG2DD/Rj5PczZBkoU0l6Kums0iZ8kwfXEt8AKAJE+hCYqbZrXK8bAKeG376adDgU1VdcNULxrLS0/V3/Qfc86QffF+YFfgvHY8/9qqWj6yonsyZF88JAzZF+cDRyS5HLgPeGtV7XBn3UP2xVuAv0/y32gGto/bEf+wTHI2zR8HC9vxmHcCOwFU1UdoxmeOAtYDdwKvG2q7O2BfSZJm0LheepIkjQmDQpLUyaCQJHUyKCRJnQwKSVIng0KaJMl9SS5OcmmSf0qy+wxv/7gkp7SPT0ryxzO5fWmmGRTSg91VVc+sqqfSfEfnDaMuSBolg0Lq9m0GJk1L8tYka9u5/P98YP1r23XfT3JWu+4l7b1SLkrylSSPH0H90q9sLL+ZLY2DJPNopn04vV0+gmaupINp5vNfleR5NHOMvR34D1V1c5LHtpv4JnBoVVWS3wX+hOYbwtKcYlBID/aIJBfTnEn8APhyu/6I9ueidnlXmuB4BnBeVd0MUFUT9wNYDHymne9/Z+DHs1O+NLO89CQ92F1V9Uxgb5ozh4kxigDvaccvnllV+1XV6R3b+VvglKp6GvB7NBPRSXOOQSFtQ3vXwD8C3tJOZX8+8F+T7AqQZFGSPYGvAa9I8rh2/cSlpwVsmcL5d5DmKC89SR2q6qIklwArquqsdorqb7ez9N4BHNvOVPou4BtJ7qO5NHUczV3VzktyK02Y7DOKY5B+Vc4eK0nq5KUnSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdfr/SVkDYHNmSdoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluate(model, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RNN's exercise 4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pretrained Glove word embeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download link:\n",
    "http://nlp.stanford.edu/data/glove.6B.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_NB_WORDS = 100000\n",
    "MAX_SEQUENCE_LENGTH = 150"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/spiros/env/lib/python3.5/site-packages/keras_preprocessing/text.py:177: UserWarning: The `nb_words` argument in `Tokenizer` has been renamed `num_words`.\n",
      "  warnings.warn('The `nb_words` argument in `Tokenizer` '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 60960 unique tokens.\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "tokenizer = Tokenizer(nb_words=MAX_NB_WORDS)\n",
    "tokenizer.fit_on_texts(mails)\n",
    "sequences = tokenizer.texts_to_sequences(mails)\n",
    "\n",
    "word_index = tokenizer.word_index\n",
    "print('Found %s unique tokens.' % len(word_index))\n",
    "\n",
    "data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "GLOVE_DIR = '/media/spiros/Data/glove.6B/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 400000 word vectors.\n"
     ]
    }
   ],
   "source": [
    "embeddings_index = {}\n",
    "f = open(os.path.join(GLOVE_DIR, 'glove.6B.300d.txt'))\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:], dtype='float32')\n",
    "    embeddings_index[word] = coefs\n",
    "f.close()\n",
    "\n",
    "print('Found %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Difference between embedding and dataset tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21955"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mail_tokens = set(tokenizer.word_index.keys())\n",
    "embedding_tokens = set(embeddings_index.keys())\n",
    "\n",
    "len(list(mail_tokens - embedding_tokens))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMBEDDING_DIM = 300\n",
    "embedding_matrix = np.zeros((len(word_index) + 1, EMBEDDING_DIM))\n",
    "for word, i in word_index.items():\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Embedding\n",
    "\n",
    "embedding_layer = Embedding(len(word_index) + 1,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(data, y, test_size=0.2, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_up(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 150)               0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, 150, 300)          18288300  \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 150, 300)          0         \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 150, 200)          320800    \n",
      "_________________________________________________________________\n",
      "bidirectional_2 (Bidirection (None, 200)               240800    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 201       \n",
      "=================================================================\n",
      "Total params: 18,850,101\n",
      "Trainable params: 561,801\n",
      "Non-trainable params: 18,288,300\n",
      "_________________________________________________________________\n",
      "Train on 2082 samples, validate on 232 samples\n",
      "Epoch 1/20\n",
      "2082/2082 [==============================] - 17s 8ms/step - loss: 0.3262 - acc: 0.8674 - val_loss: 0.1649 - val_acc: 0.9483\n",
      "Epoch 2/20\n",
      "2082/2082 [==============================] - 15s 7ms/step - loss: 0.1456 - acc: 0.9385 - val_loss: 0.1930 - val_acc: 0.9440\n",
      "Epoch 3/20\n",
      "2082/2082 [==============================] - 16s 8ms/step - loss: 0.1132 - acc: 0.9616 - val_loss: 0.0772 - val_acc: 0.9698\n",
      "Epoch 4/20\n",
      "2082/2082 [==============================] - 16s 7ms/step - loss: 0.0806 - acc: 0.9765 - val_loss: 0.0602 - val_acc: 0.9784\n",
      "Epoch 5/20\n",
      "2082/2082 [==============================] - 15s 7ms/step - loss: 0.0544 - acc: 0.9841 - val_loss: 0.0779 - val_acc: 0.9698\n",
      "Epoch 6/20\n",
      "2082/2082 [==============================] - 15s 7ms/step - loss: 0.0298 - acc: 0.9885 - val_loss: 0.0630 - val_acc: 0.9698\n",
      "Restoring model weights from the end of the best epoch\n",
      "Epoch 00006: early stopping\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Model\n",
    "\n",
    "inputs =  Input(shape=(150, ), dtype='int32')\n",
    "embeddings = embedding_layer(inputs)\n",
    "dropped_embeddings = Dropout(0.0)(embeddings)\n",
    "lstms_1 = Bidirectional(LSTM(100, return_sequences=True))(dropped_embeddings)\n",
    "lstms_2 = Bidirectional(LSTM(100))(lstms_1)\n",
    "outputs = Dense(1, activation='sigmoid')(lstms_2)\n",
    "\n",
    "# Wrap model\n",
    "model = Model(inputs=inputs, outputs=outputs)\n",
    "model.summary()\n",
    "\n",
    "es = EarlyStopping(monitor='val_loss', patience=2, restore_best_weights=True, verbose=1)\n",
    "\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "history = model.fit(X_train, y_train, epochs=20, batch_size=64, verbose=1, validation_split=0.1, callbacks=[es])\n",
    "\n",
    "y_pred = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "579/579 [==============================] - 3s 5ms/step\n",
      "LOSS ON TEST DATA=0.08231489093130172\n",
      "ACCURACY ON TEST DATA=0.9792746113989638\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VfWd//HXJ3vIQsgGJAHCEpawJRBQi8qqIlYWa1XU1p3quNT66G9ql5l2nHbGbha1tiPu2iq1WpCqFDdQcAUkIHsiiyRASMKSkJD98/vjnIQLBhLCvblJ7uf5eOSR3LPdz8U273yX8z2iqhhjjDGnE+TvAowxxnR8FhbGGGNaZGFhjDGmRRYWxhhjWmRhYYwxpkUWFsYYY1pkYWHMWRCRdBFREQlpxbE3isiqs72OMf5gYWEChojsEpEaEUk8afs69xd1un8qM6bjs7AwgWYnMLfxhYiMBLr5rxxjOgcLCxNoXgC+6/H6BuB5zwNEpLuIPC8ixSKyW0R+JiJB7r5gEfmdiJSIyA7gsmbOfUpE9olIoYj8UkSCz7RIEUkRkSUiclBE8kXkNo9940VkjYiUiUiRiDzkbo8Qkb+ISKmIHBaR1SLS80zf25jmWFiYQPMJECsiw9xf4tcAfznpmEeB7sAAYCJOuNzk7rsN+CaQDeQAV5507rNAHTDIPeZi4NY21LkQKABS3Pf4HxGZ4u57GHhYVWOBgcDL7vYb3Lr7AAnA7cCxNry3MV9jYWECUWPr4iJgC1DYuMMjQH6squWqugv4PfAd95CrgPmqukdVDwL/63FuT2AGcK+qVqjqAeAP7vVaTUT6ABOAH6lqlarmAk9yvEVUCwwSkURVPaqqn3hsTwAGqWq9qq5V1bIzeW9jTsXCwgSiF4BrgRs5qQsKSARCgd0e23YDqe7PKcCek/Y16ueeu8/tBjoMPA4kn2F9KcBBVS0/RQ23AIOBrW5X0zc9PtcyYKGI7BWR34hI6Bm+tzHNsrAwAUdVd+MMdM8A/nHS7hKcv9D7eWzry/HWxz6cbh7PfY32ANVAoqrGuV+xqjr8DEvcC8SLSExzNahqnqrOxQmhXwOviEiUqtaq6n+paibwDZzusu9ijBdYWJhAdQswRVUrPDeqaj3OGMCvRCRGRPoB93F8XONl4B4RSRORHsD9HufuA94Cfi8isSISJCIDRWTimRSmqnuAj4D/dQetR7n1/gVARK4XkSRVbQAOu6c1iMhkERnpdqWV4YRew5m8tzGnYmFhApKqfqmqa06x+26gAtgBrAJeBJ529z2B09WzHvicr7dMvguEAZuBQ8ArQO82lDgXSMdpZSwCfq6q77j7pgObROQozmD3Nap6DOjlvl8ZzljM+zhdU8acNbGHHxljjGmJtSyMMca0yMLCGGNMiywsjDHGtMjCwhhjTIu6zHLIiYmJmp6e7u8yjDGmU1m7dm2Jqia1dFyXCYv09HTWrDnVTEhjjDHNEZHdLR9l3VDGGGNawcLCGGNMiywsjDHGtKjLjFk0p7a2loKCAqqqqvxdSpcSERFBWloaoaG2oKkxgaJLh0VBQQExMTGkp6cjIv4up0tQVUpLSykoKKB///7+LscY0066dDdUVVUVCQkJFhReJCIkJCRYa82YANOlwwKwoPAB+zc1JvB0+bBoSX1DA/uPVFFdW+/vUowxpsMK+LBoUCg5Wk1RebXXr11aWkpWVhZZWVn06tWL1NTUptc1NTWtusZNN93Etm3bvF6bMcaciS49wN0aocFBJESHUVJeTVVMOBGhwV67dkJCArm5uQD84he/IDo6mh/+8IcnHKOqqCpBQc3n9jPPPOO1eowxpq0CvmUBkBQdjohwoMz7rYvm5Ofnk5mZyXXXXcfw4cPZt28f8+bNIycnh+HDh/PAAw80HXv++eeTm5tLXV0dcXFx3H///YwePZrzzjuPAwcOtEu9xhgTMC2L//rnJjbvLTvl/pr6BmrrGogMCyaolQO4mSmx/Pzy4W2qZ+vWrTz//PPk5OQA8OCDDxIfH09dXR2TJ0/myiuvJDMz84Rzjhw5wsSJE3nwwQe57777ePrpp7n//vubu7wxxniVtSxcocFBIFBb3z7Ptx84cGBTUAC89NJLjBkzhjFjxrBlyxY2b978tXMiIyO59NJLARg7diy7du1ql1qNMSZgWhataQHsP3KMA+XVZPSMIdKLYxfNiYqKavo5Ly+Phx9+mM8++4y4uDiuv/76Zu9jCAsLa/o5ODiYuro6n9ZojDGNrGXhITE6nGARDpS17w1nZWVlxMTEEBsby759+1i2bFm7vr8xxrQkYFoWrRESHERCdDgHyqs4VlNPZJhvWxeNxowZQ2ZmJkOHDqVfv35MmDChXd7XGGNaS1TV3zV4RU5Ojp788KMtW7YwbNiwM7pOXX0D24rKiQoLIT0xquUTAlRb/m2NMR2PiKxV1ZyWjrNuqJOEBAeRGB1OWVUtx2psTMAYY8DColmJ0WEEBwlF7XTfhTHGdHQWFs0IDjreuqi01oUxxlhYnEpj66K97uo2xpiOzMLiFIKDgkhqbF1UW+vCGBPYLCxOIyE6nJAg8cmKtMYY05n4NCxEZLqIbBORfBH52iJGInK7iHwhIrkiskpEMj32/dg9b5uIXOLLOk8lOEhIjAmnvKqWija2LiZPnvy1m+zmz5/PHXfcccpzoqOjAdi7dy9XXnlls8dMmjSJk6cKn2z+/PlUVlY2vZ4xYwaHDx9ubenGGNPEZ2EhIsHAY8ClQCYw1zMMXC+q6khVzQJ+AzzknpsJXAMMB6YDf3Kv1+4SosIJCQqiqI13dc+dO5eFCxeesG3hwoXMnTu3xXNTUlJ45ZVX2vS+8PWwePPNN4mLi2vz9YwxgcuXLYvxQL6q7lDVGmAhMMvzAFX1XAY2Cmi8Q3AWsFBVq1V1J5DvXq/dBQcJSTFhHK2ua1Pr4sorr+SNN95oetjRrl272Lt3L9nZ2UydOpUxY8YwcuRIXnvtta+du2vXLkaMGAHAsWPHuOaaaxg2bBhz5szh2LFjTcfdcccdTcub//znPwfgkUceYe/evUyePJnJkycDkJ6eTklJCQAPPfQQI0aMYMSIEcyfP7/p/YYNG8Ztt93G8OHDufjii094H2NM4PLlch+pwB6P1wXAOScfJCJ3AvcBYcAUj3M/Oenc1GbOnQfMA+jbt+/pq1l6P+z/otXFe0pE6VZT7yxd7rnAYK+RcOmDpz03Pj6e8ePHs3TpUmbNmsXChQu56qqriIyMZNGiRcTGxlJSUsK5557LzJkzT/l86z//+c9069aNLVu2sGHDBsaMGdO071e/+hXx8fHU19czdepUNmzYwD333MNDDz3E8uXLSUxMPOFaa9eu5ZlnnuHTTz9FVTnnnHOYOHEiPXr0IC8vj5deeoknnniCq666ildffZXrr7++Tf9uxpiuw+8D3Kr6mKoOBH4E/OwMz12gqjmqmpOUlOSbAgFBCA0Oor5BqW/D8iieXVGNXVCqyk9+8hNGjRrFtGnTKCwspKio6JTX+OCDD5p+aY8aNYpRo0Y17Xv55ZcZM2YM2dnZbNq0qdnlzT2tWrWKOXPmEBUVRXR0NFdccQUrV64EoH///mRlZQG2DLox5jhftiwKgT4er9PcbaeyEPhzG89tWQstgJaENCj5ReWEhwQxMCn6jM6dNWsWP/jBD/j888+prKxk7NixPPvssxQXF7N27VpCQ0NJT09vdlnyluzcuZPf/e53rF69mh49enDjjTe26TqNwsPDm34ODg62bihjDODblsVqIENE+otIGM6A9RLPA0Qkw+PlZUCe+/MS4BoRCReR/kAG8JkPa21RUJCQHBNORXUdR6tqz+jc6OhoJk+ezM0339w0sH3kyBGSk5MJDQ1l+fLl7N69+7TXuPDCC3nxxRcB2LhxIxs2bACc5c2joqLo3r07RUVFLF26tOmcmJgYysvLv3atCy64gMWLF1NZWUlFRQWLFi3iggsuOKPPZIwJLD5rWahqnYjcBSwDgoGnVXWTiDwArFHVJcBdIjINqAUOATe4524SkZeBzUAdcKeq1vuq1taK7xZGcXk1RWXVRIWHnHJ8oTlz585lzpw5Td1R1113HZdffjkjR44kJyeHoUOHnvb8O+64g5tuuolhw4YxbNgwxo4dC8Do0aPJzs5m6NCh9OnT54TlzefNm8f06dNJSUlh+fLlTdvHjBnDjTfeyPjxzpyBW2+9lezsbOtyMsacki1RfoZKjlaz9/Ax+idGERMR6tVrdya2RLkxXYMtUe4j8d3CCA0Ooqismq4StMYY0xILizPUOHZRWVPHUVszyhgTILp8WPjir/8eUWGEBXDrIhA/szGBrkuHRUREBKWlpV7/5RYkQpLbuigPsNaFqlJaWkpERIS/SzHGtCNf3mfhd2lpaRQUFFBcXOz1a6sqpWXVHNoLyTGB9YszIiKCtLQ0f5dhjGlHXTosQkND6d+/v8+u/8Xqr/jRq1/w1A05TB3W02fvY4wx/talu6F87YoxafSN78Yf3tlu/fjGmC7NwuIshAYHcfeUQWwsLOPtzade18kYYzo7C4uzNCc7lfSEbvzhnTwaGqx1YYzpmiwszlJIcBB3T8lgy74y3tq839/lGGOMT1hYeMGsrBQGJEYx31oXxpguysLCC0KCg7hnagZb95ezdKO1LowxXY+FhZdcPjqFgUlRPPzudmtdGGO6HAsLLwkOEr4/bTDbi47yxhf7/F2OMcZ4lYWFF102sjcZydHMf2c79da6MMZ0IRYWXhQcJNw7bTBfFlfw+oa9/i7HGGO8xsLCyy4d0YuhvWJ4+J086uob/F2OMcZ4hYWFlwUFCfdOy2BHSQVL1lvrwhjTNVhY+MDFmb0Y1juWR9611oUxpmuwsPCBxtbFrtJKFq0r9Hc5xhhz1iwsfOTizJ4MT4nl0ffyqbXWhTGmk7Ow8BER4QfTBvPVwUr+8XmBv8sxxpizYmHhQ1OHJTMqrTuPvpdPTZ21LowxnZeFhQ81ti4KDh3jVWtdGGM6MZ+GhYhMF5FtIpIvIvc3s/8+EdksIhtE5F0R6eexr15Ect2vJb6s05cmDUkiq08cf7TWhTGmE/NZWIhIMPAYcCmQCcwVkcyTDlsH5KjqKOAV4Dce+46papb7NdNXdfqaiPCDiwZTePgYL6/Z4+9yjDGmTXzZshgP5KvqDlWtARYCszwPUNXlqlrpvvwESPNhPX5zYUYiY/rG8djyfKrr6v1djjHGnDFfhkUq4PmndIG77VRuAZZ6vI4QkTUi8omIzG7uBBGZ5x6zpri4+Owr9hER4b6LhrDvSBV/W22tC2NM59MhBrhF5HogB/itx+Z+qpoDXAvMF5GBJ5+nqgtUNUdVc5KSktqp2raZMCiBcek9eGx5PlW11rowxnQuvgyLQqCPx+s0d9sJRGQa8FNgpqpWN25X1UL3+w5gBZDtw1p9rnFmVFFZNS999pW/yzHGmDPiy7BYDWSISH8RCQOuAU6Y1SQi2cDjOEFxwGN7DxEJd39OBCYAm31Ya7s4b2AC5/SP508rvrTWhTGmU/FZWKhqHXAXsAzYArysqptE5AERaZzd9FsgGvj7SVNkhwFrRGQ9sBx4UFU7fVg0zowqLq/mr59a68IY03mIatd4oltOTo6uWbPG32W0yrVPfML2oqOs/PfJRIYF+7scY0wAE5G17vjwaXWIAe5A84OLBlNytJq/fLLb36UYY0yrWFj4wbj0eM4flMj/vf8llTV1/i7HGGNaZGHhJz+4KIPSihqe/9haF8aYjs/Cwk/G9ovnwsFJPP7+lxytttaFMaZjs7Dwox9My+BQZS3PfbTL36UYY8xpWVj4UXbfHkweksQTK3dQXlXr73KMMeaULCz87N5pgzlcWcuzH+7ydynGGHNKFhZ+NrpPHNOGJfPEyh2UWevCGNNBWVh0APdOG0xZVR3PrNrl71KMMaZZFhYdwIjU7lyc2ZMnV+3gyDFrXRhjOh4Liw7i3mmDKa+q46lVO/1dijHGfI2FRQeRmRLL9OG9eHrVTg5X1vi7HGOMOYGFRQdy70UZHK2u48mV1rowxnQsFhYdyNBesVw2sjfPfLiTQxXWujDGdBwWFh3M96dlUFlbz4KVO/xdijHGNLGw6GAG94zhm6NSeO6jXZQerW75BGOMaQcWFh3Q96dmUFVbz4IPrHVhjOkYLCw6oEHJ0cwcncLzH++mxFoXxpgOwMKig7pnagbVdfU8/v6X/i7FGGMsLDqqAUnRzM5O5YVPdnOgvMrf5RhjApyFRQd2z5QMauuV/1thYxfGGP+ysOjA0hOjuCI7lb98upuiMmtdGGP8x8Kig7t7SgYNDcqfV9jYhTHGfywsOri+Cd341pg0XvzsK/YdOebvcowxAcqnYSEi00Vkm4jki8j9zey/T0Q2i8gGEXlXRPp57LtBRPLcrxt8WWdHd9eUQTQ0KH9abq0LY4x/+CwsRCQYeAy4FMgE5opI5kmHrQNyVHUU8ArwG/fceODnwDnAeODnItLDV7V2dH3iu/HtnD78bfUeCg9b68IY0/582bIYD+Sr6g5VrQEWArM8D1DV5apa6b78BEhzf74EeFtVD6rqIeBtYLoPa+3w7poyCEV5bHm+v0sxxgQgX4ZFKrDH43WBu+1UbgGWnsm5IjJPRNaIyJri4uKzLLdjS42L5Opxffj7mj0UHKps+QRjjPGiDjHALSLXAznAb8/kPFVdoKo5qpqTlJTkm+I6kDsnD0IQa10YY9qdL8OiEOjj8TrN3XYCEZkG/BSYqarVZ3JuoOndPZK54/vw9zUF7DlorQtjTPvxZVisBjJEpL+IhAHXAEs8DxCRbOBxnKA44LFrGXCxiPRwB7YvdrcFvH+bPIigIOHR9/L8XYoxJoD4LCxUtQ64C+eX/BbgZVXdJCIPiMhM97DfAtHA30UkV0SWuOceBP4bJ3BWAw+42wJez9gIrjunL69+Xsiukoozv0D5fljzDCy+Ew5/5f0CjTFdkqiqv2vwipycHF2zZo2/y2gXB8qquOA3y/nmqBR+f9Xo0x+sCsVbYesbsO1NKFzr7hDoPQpuXgahkT6v2RjTMYnIWlXNaem4kPYoxnhXcmwE15/bj2c+3MldUwbRPzHqxAPq62DPJ7D1TScgDu10tqeMgSk/gyEz4PAeeOlqeP0+mP0nEGn/D2KM6TRaFRYiMhAoUNVqEZkEjAKeV9XDvizOnNrtEwfy109388i7efzh6iyoPgpfvueEw/ZlcOwgBIdB/4nwjbthyKUQm3L8Aj2Hw8T74f0HIW0sjLvVfx/GGNPhtbZl8SqQIyKDgAXAa8CLwAxfFWZOLykmnDvGRlO0+nkqKv6LqMIPob4aIuJg8CVO62HQVAiPOfVFJv4I9n4OS++HXqOgz/j2+wDGmE6ltWHRoKp1IjIHeFRVHxWRdb4szDTjpPGH7xeuhVAo2debqHG3OAHR9zwIbuV/1qAguGIBLJgEL38X5r0PMT19+hGMMZ1Ta8OiVkTmAjcAl7vbQn1TkjlBfR3s+dTpXtr6xtfGH54qHsYv18Bb2RPJ6HmaVsSpRPaAq/8KT06Dv98INyyBYPtPa4w5UWvD4ibgduBXqrpTRPoDL/iurAB3yvGHC782/jCnooaH1r/Hw+/m8cdrx7Tt/XqNgJmPwj9uhbf+Ay590IsfxhjTFbQqLFR1M3APgHuTXIyq/tqXhQWc8v2wbakTEDveb/X4Q3xUGDdOSOdPK77k7v3lDOnVhtYFwKhvO+MXn/wJUsc6r40xxtXa2VArgJnu8WuBAyLyoare58PaurbG8YdtbzpTXAvde0Ti+sG4W5zWQ9/zWtUldNsFA3juo908/O52/nTd2LbXdNEDsG89LLkbkoc5LQ5jjKH13VDdVbVMRG7FmTL7cxHZ4MvCuqRTjj9kw+SfwdAZkJx5xvc8xHUL4+YJ6TzyXj6b95aRmRLbtvqCQ+HKZ2DBRPjbdTBvhTOmYYwJeK0NixAR6Q1chbPon2mtMxh/OBu3nD+AZz7axcPvbufx77R4M+apxfSEq56HZ2bAq7fBtS87s6aMMQGttWHxAM4aTx+q6moRGQDYSnanUl4E25c63Us7Vpw0/nApDJp2+vsf2qB7t1BuntCfh9/NY2PhEUakdm/7xfqMh0t/DW/c59y0N/kn3ivUGNMp2dpQ3qAKxdtg2xsnjT/0hSGXOd1LrRx/OBtHjtVywa/fY3z/BJ684SxaF+B8ptfuhNy/wty/wZCAflChMV2WV9eGEpE04FFggrtpJfB9VS1oe4mdnOf4w7Y34eAOZ/tZjj+cje6Rodx6wQAeens7XxQcYWTaWbQuROCy30PRRvjHPJi3HBIGeq9YY0yn0qqWhYi8jbO8R+O9FdcD16nqRT6s7Yy0S8uipsIZf9j6Jmz/14njD0Mudaa4emH84WyUV9Vy/q+XM7ZfD56+cdzZX/DwV/D4RIjpBbe8DeHRZ39NY0yH4e1VZ5NU9RmP18+KyL1tK62TaXb8oTtkXOK0HgZOhYg2zj7ygZiIUOZdOIDfLttG7p7DZPWJO7sLxvWFK5+Cv3zLmVJ75dO2Qq0xAai1YVHqPif7Jff1XKDUNyX5mef4w7alULAGUOeXZs7N7Tb+cDZu+EY6T67cwfx3tvPsTV5YHHDgFJj6n/DOLyAtB8678+yvaYzpVFobFjfjjFn8AVDgI+BGH9XU/hrqnfGHxgcEnTD+8FOni6nn8E7zF3V0eAjzLhzIr/+1lbW7DzG2nxfulZhwr/PgpLf+w1mhtv8FZ39NY0yn0ebZUCJyr6rO93I9bdbmMYtDu2DBZGf8ISjUGX8YOgMGXwrdU71eZ3upqK7jgt8sZ3hKLC/cco53LlpVBk9OhcqD8L0POvW/jzHG0doxi7O526prLPXRvS8Mnw3ffhb+fQd85x/Og4A6+S/CqPAQbp84gJV5JazZ5aXHl0fEOivU1lXDy99xvhtjAsLZhEXn6JNpSVAQfPMPMHxOhxqo9obrz+1HYnQYf3hnu/cumjQY5vzZ6ZJa+u/eu64xpkM7m7DoGnfzdWHdwkK4feJAPswv5dMdXpyPMOxyOP8HsPZZ+Px5713XGNNhnTYsRKRcRMqa+SoH/HtDgWmV687pR1JMuHdbFwBT/gMGTII3fui0MowxXdppw0JVY1Q1tpmvGFVt7Uwq40eRYcHcMXEgn+w4yEdflnjvwkHB8K2nIbon/O27UOHFaxtjOhxbTjQAXHtOX3rGhjP/7Ty8uhZYVAJc/QJUlsArNzlLoBhjuiQLiwAQERrMv00axGe7DvLRl16+lzIlCy57CHZ+AO894N1rG2M6DJ+GhYhMF5FtIpIvIvc3s/9CEflcROpE5MqT9tWLSK77tcSXdQaCq8f1oVdsBH94e7t3WxcA2ddBzi3w4cOwaZF3r22M6RB8FhYiEgw8BlwKZAJzRSTzpMO+wrkT/MVmLnFMVbPcr5m+qjNQRIQGc+eUQazZfYiVeT4YX5j+IKSNg8V3woGt3r++McavfNmyGA/kq+oOVa0BFgKzPA9Q1V2qugFo8GEdxnVVThop3SP4wzs+aF2EhDlP2AuLch7JWnXEu9c3xviVL8MiFdjj8brA3dZaESKyRkQ+EZHZzR0gIvPcY9YUFxefTa0BITwkmLumZLDuq8Os2O6Df6/YFLjqOWcJlUV3QIP9DWBMV9GRB7j7ueuVXAvMF5GvPXlHVReoao6q5iQlJbV/hZ3QlWPTSI2LZL4vxi4A+n0DLv6ls2rvqoe8f31jjF/4MiwKgT4er9Pcba2iqoXu9x3ACiDbm8UFqrCQIO6eMoj1BUeY9diHPL1qJwfKq7z7JufcDiO/De/9EvLf8e61jTF+4cuwWA1kiEh/EQkDrgFaNatJRHqISLj7cyLO41w3+6zSAPPtnD785zczqatXHnh9M+f+z7t856lPeXVtAUervXCvhAhc/rCzrPsrtzjdUsaYTq3NS5S36uIiM4D5QDDwtKr+SkQeANao6hIRGQcsAnoAVcB+VR0uIt8AHscZ+A4C5qvqU6d7r3Z5rGoXlFdUzuLcQl7L3UvBoWOEhwQxLbMns7NSmTg4ibCQs/h74uAOWDDJeXDUzW9BWDev1W2M8Y7WLlHu07BoTxYWZ0dVWbv7EItzC3ljwz4OVdYS1y2UGSN7MzsrlZx+PQgKasNCw9vfghevglFXw5z/6zQPkDImUFhYmDarrW/gg+3FvJa7l7c276eqtoHUuEhmZqUwOyuVIb1izuyCK34NK/4HZvwOxt/mm6KNMW1iYWG8oqK6jrc272fxur2syi+hvkEZ2iuG2dmpzBydQkpcZMsXaWiAhXOdwe4b34C+5/q+cGNMq1hYGK8rLq/mjQ17WZy7l9w9hxGB8enxzM5OZcaI3nTvFnrqk48dhicmQ02F80jWmF7tV7gx5pQsLIxP7Sqp4LXcvbyWW8iOkgpCg4VJQ5KZnZXK1GHJRIQGf/2kos3OM7x7jYIb/unc9W2M8SsLC9MuVJWNhWUszi1kyfq9FJdXExMewiUjejE7K5XzBiYQ7DkwvvFVeOVmGP89mPEb/xVujAEsLIwf1DcoH39ZyuLcQv61cT9Hq+tIjgnn8tHOwPiI1FhEBJb9FD7+I8x5HEZf4++yjQloFhbGr6pq63l3ywEW5xayYtsBauuVAUlRzM5KZdaoZPq9cR0UrIZb3oLeo/1drjEBy8LCdBiHK2tYunE/i9cV8unOgwBMSoPHyu8lIiKC4O+9D93i/VylMYHJwsJ0SHsPH2PJ+r0sXldIRNE6/hb2ANsis/jy4qe5eHgqUeH2aHdj2pOFhenwtu0vZ+eyPzJ954M8UjebP8tcLsrsyezsFC7ISCI0uCMvimxM19DasLA/44zfDOkVw5Dv3o8uKeSedS/QI/0cfp8XxJL1e4mPCuOykb2ZnZ3CmL49nIFxY4zfWMvC+F9tFTwzHUryqbnlXT4ojWNxbiFvby6iuq6BPvGRzBqdyuzsFAYln+FSI8aY07JuKNO5HN4DCyZCVBLc+i6ER3O0uo5lG/ezOLeQD/NLaFDI7B3L7OwUZo5OpVf3CH9XbUynZ2FhOp8d78MLs2HYTPj2syesUHuDV0UPAAAVoUlEQVSgvIrX1+/jtdxC1hccQQTO7Z/A7OwUpo/oTffI0yw1Yow5JQsL0zl9+DC8/Z9w0X/DhHuaPWRnSQWvuc/g2FlSQVhIEFOGJDM7O4VJQ06x1IgxplkWFqZzUoW/3wBb/gnfWQwDJp7mUGVDwREW5xbyz/X7KDlaTUxECDNG9GZWdgrn9k9o2zM4jAkgFham86ouhyenQUUxzHsf4vq0eEpdfQMfuUuNLNu4n4qaenrFRjAzK4WZo1MYnhJrM6qMaYaFhencSvJgwWRIzICblkJo6wezj9XU886WIl7L3cv7252lRgYlRzM7K4VZWan0ibfHuxrTyMLCdH5bXoe/XQdjvgszH23TJQ5V1PDmxn28tm4vn+1ylhrJ6deD2dmpXDayNz2ibJl0E9gsLEzX8O5/w8rfweUPw9gbz+pSBYcqeS3XWWok78BRQoOFiYOTmZN9mmdwGNPFWViYrqGhHv76bdi1Em76F6SNPetLqiqb95U1PbypqMx5Bsf0Eb2YnZ3KuQNOegaHMV2YhYXpOioPwoJJ0FDnDHhHJ3nt0vUNyqc7Slm0znkGR3l1HT1jw5mVlcqsrBQye9vAuOnaLCxM17JvPTx1MaSNc6bUBnt/WbPGZ3AsWlfYNDA+uGd0U3Ck9bCBcdP1WFiYrmf9Qlj0PTjvLrjkVz59q0MVNbzxhXPH+OpdhwAYnx7P7OxUZozsRVw3Gxg3XUOHCAsRmQ48DAQDT6rqgyftvxCYD4wCrlHVVzz23QD8zH35S1V97nTvZWERIN78f/DZArjyaRjxrXZ5yz0HK1myfi+L1hWS7w6MTx6SzOzsVKYMtYFx07n5PSxEJBjYDlwEFACrgbmqutnjmHQgFvghsKQxLEQkHlgD5AAKrAXGquqhU72fhUWAqKuB5y6H/RucBQd7ZrbbW6sqm/aWsXhdIUvW7+VAud0xbjq/jvA8i/FAvqrucAtaCMwCmsJCVXe5+xpOOvcS4G1VPejufxuYDrzkw3pNZxASBlc9B49fCH+7Hm57DyLj2uWtRYQRqd0ZkdqdH88YxsdfOgPjb3yxj7+t2UPv7hHMHJ3C7OxUhvWObZeajGkvvgyLVGCPx+sC4JyzODf15INEZB4wD6Bv375tq9J0PjG94NvPwXPfhEW3wzUvQlD7PlUvOEg4PyOR8zMS+WXNCPeO8UKeWrWTxz/YwZCeMczOdgbGU+Ii27U2Y3yhUz+3UlUXqGqOquYkJXlvOqXpBPqdB5f8L2xf6ty050eRYcFcPjqFJ28Yx2c/ncZ/zx5BdEQIv/7XVr7x4Htc/fjHLPzsK44cq/VrncacDV+2LAoBzxXg0txtrT130knnrvBKVabrGH8bFK6F5f8DKdmQcZG/KyI+KozvnNuP75zbj69KK3ktt5BFuYXc/48v+M/XNjFlqLOU+uShyYSH2MC46Tx8OcAdgjPAPRXnl/9q4FpV3dTMsc8Cr580wL0WGOMe8jnOAPfBU72fDXAHqJpK5/6LI1/BvBUQP8DfFX2NqrKxsIxF7sB4ydFqYiNCmDGyN7OzUxmfHm8D48Zv/D4byi1iBs7U2GDgaVX9lYg8AKxR1SUiMg5YBPQAqoD9qjrcPfdm4CfupX6lqs+c7r0sLALYwZ3OHd7d0+CWtyGs494819xS6indI5iZ5TxjfGgvGxg37atDhEV7srAIcHnvwF+vhJFXwhVPnPBI1o7qWE09b28pYvG6Qj7YXkxdgzK0VwxzslOZmZVC7+42MG58z8LCBJ4Pfgvv/RKm/xrOvd3f1ZyR0qPVvPHFPhatK2TdV4ftGeOm3VhYmMDT0ODce5G3DG74J/T7hr8rapPdpRUsXuesiLvDfcb41KHOHeOThiTZwLjxKgsLE5iqjsATU6CqDL73AcT29ndFbdb4jPFF6wp5fcNeSo7W0D0y1BkYz0phnA2MGy+wsDCB68AWeGIq9BwON77h3PXdydXVN7Aqv4TXcveybNN+KmvqSY2LZFaWc8f44J4x/i7RdFIWFiawbVoEf78Rxt0Kl/3e39V4VWVNHW9vLmLRukJW5pVQ36Bk9o5ldnYKM0en0qt7659XboyFhTFv/Qd89AjM/jNkXevvanyi5Gg1r6/fy+LcveTucQbGzxuQwOzsVKaP6EVshA2Mm9OzsDCmvg5emA0Fq+HmZZCS5e+KfGpnSQWv5RayeF0hu0ormwbGpwxNZuKQJJJjrMVhvs7CwhiAihJ4fCJIEHzvfegW7++KfE5VWV9whMXrClm6cR9FZdUAjEiNZdLgZCYPTSKrTw97zrgBLCyMOa5wLTw9HfqcAxf+P+g1MiBCA5zg2LKvnOXbDvD+tmLWfnWI+gale2QoFw5OYtLgJCYOSSIxOtzfpbZNXbUzoWH/BufRu/s2ODPiEjMgaSgkDXG+EjI69J39/mRhYYyndX+BJXeDuo9OiU11QqPnCOd7r5HQo3+7L3Xe3o4cq2VVXokTHtuLKS53Wh2j0rozaUgyk4YkMTotrmO2OmoqYP9GNxhynXA4sBUa3NV8w6Kh1yiI7AEl2+HgDtB692SBuL5ugAx2vicOcX6O6O63j9QRWFgYc7KKUucXzf4voGij87142/FfKGHRznTbxvDoORKSh3XZv0gbGpTN+8pYse0Ay7cVs+6rQzQo9OjmtDomD0nmwsFJxEf5YerxsUNOK2Hf+uOthpI8nAdnAt0SnGDoPRp6j4LeWV8P+7oaOPglFG+F4u3O95LtznXqq48fF9PbbYEMhcTBx1skUYnt+pH9xcLCmNaorYLiLe5frF8cD5LqMme/BDldGL1GnBgiMT39W7cPHK6s4YO8Ela4XValFTWIwOi0OCYNccJjZGp3798IWL7/eDDsy3XC4fBXx/fHpjqh4BkOsaltX/+roR4O7XL+UGgMkMZAqa04fly3BLf1MeTEFklM706x9lhrWVgY01aqcHj38fBoDJIjHr/AopKPh0fjV8IgCOoaS3E0NCgb9x5h+dZiVmw/QO6ew6hCQlQYE91xjgszkuhxJq2Oxn/XxrGFxlbD0aLjx8QPOCkYRrffX/gNDVBWCCXb3CDZdjxQqg4fPy481m2BuEHSGChx/TplN6aFhTHeduwQFG3yCJEvnMHVxj7zkEin28ozQHoOh/DOf3f1wYoaVuYVs3zrAT7IK+FgRQ1BAll94pg8JJlJQ5IZnhJ7vNXRUA+l+W4wrD8eDFVHnP0S7PyV3nvU8XDoNRIiOuAS7apQUey2PtwAaQwUz6ALiYTEQR7jIe5X/AAI7rj3u1hYGNMe6mqcboymcRB3TOTYoePHxA843n3VGCKxKZ22K6O+QdlQcJgV24pZse0AWwtLGEQB50YWMKX7PobJLnqUb0dqK50TgsOd0GwaXxgNyZkQ2gWWYD926MTxkMbuLM9WaFCI0+r0HA9JGuJs6wD/BhYWxviLqtOd0TQOssEJkoM7jh8T2cMNjlHHZ2UlDenQf4ECx2ck7VsP+50Wgx7Yiritq6NEsqmhH5s1nYr4TBIGjWdU9jgyUxOQThqObVJ9FErzvt6ddWjn8Rl5CPRIb36GVju2Ri0sjOloqsub6cbaDHVVzv7gMOcXxgndWCMgMs4/9bZmRtJJ4wv1cenkFjgzrFZsK+aLQqfbKTkmnElDkpg0JJnzMxIDdxmSumqne+7k7qzSfKivOX5cbOqJ4yGNLRIf3B9kYWFMZ1Bf50zvbGyBNN5HUFF8/JjufT0CxJ2VFdfPu91YrZ2R5BkOrehKO1BexQfbnfs6PtheTHlVHSFBwph+PdyxjiSG9ooJrFZHc+rr3BlaW08cYC/ZDo3deQBRSSeOhzQGSXTPNv/vwcLCmM6svAiKvjixFVKaf7wLIzz2xBsKe42ApGEQ2sL6T62akTTwxIFnL81IqqtvYN2ewyzf6rQ6Nu9zpif3io1oanVMGJRATKC2OprT0ABlBSd2ZTW2SBonCwCk5sBt77bpLSwsjOlqaiqPL23ReFPh/o3H7w2QYOcvTc8706OTna6v085I8hh47jmi3WYkFZVV8f62YpZvO8CqvBLKq51Wx7j0eOe+jqHJZCRHW6ujOapOwDeGSEgYjL2xTZeysDAmEDQ0OIOmni2Qoo3OALunE2YkueHQgWYk1dY3sHb3oeMzrPaXA5DSPYJJQ5OZNDiJCYMSiQoP8XOlXY+FhTGBrKLU6cY6Wgw9M51pmx19ppWHfUeONQXHqrwSKmrqCQsOYlz/42MdA5Os1eENFhbGmC6hpq6BNbsPNoXH9qKjAKT1iGxahuS8gQl0C7NWR1tYWBhjuqTCw8ecxQ+3FvPRlyVU1tQTFhLEOf3jm1bOHZAYZa2OVuoQYSEi04GHgWDgSVV98KT94cDzwFigFLhaVXeJSDqwBdjmHvqJqt5+uveysDAm8FTX1bN65yF35dwDfFnsDPandI/g/IxEzs9IYsLABBI66/M62oHfw0JEgoHtwEVAAbAamKuqmz2O+TdglKreLiLXAHNU9Wo3LF5X1RGtfT8LC2PMnoOVfJBXzKq8Ej7ML6Gsqg6AzN6xXJCRyPkZiYxLjycitGss+OgNHSEszgN+oaqXuK9/DKCq/+txzDL3mI9FJATYDyQB/bCwMMachfoG5YvCI6zKK2ZlXgmff3WI2nolPCSIcenxTstjUCKZvWO9v+x6J9LasPDliFAqsMfjdQFwzqmOUdU6ETkCJLj7+ovIOqAM+Jmqrjz5DURkHjAPoG/fvt6t3hjTqQUHCVl94sjqE8ddUzKoqK7js50HWZlXwqr8Yh5cuhWA+KgwvjEwwW15JJEa1zGmE3c0HXX6wD6gr6qWishYYLGIDFfVMs+DVHUBsACcloUf6jTGdBJR4SFMHprM5KHJABwoq2JVfgmr8kpYmV/C6xv2ATAgMaqp1XHeQLujvJEvw6IQ6OPxOs3d1twxBW43VHegVJ2+sWoAVV0rIl8CgwHrZzLGeEVybARXjEnjijFpqCrbi46yMq+YVfkl/H1NAc9/vLupdXL+IGe8I6tPHKHBne8BR97gyzGLEJwB7qk4obAauFZVN3kccycw0mOA+wpVvUpEkoCDqlovIgOAle5xB0/1fjZmYYzxluq6etZ9dbip1fFFwWEaFKLDQzh3QLwbHkkMTOr8U3T9PmbhjkHcBSzDmTr7tKpuEpEHgDWqugR4CnhBRPKBg8A17ukXAg+ISC3QANx+uqAwxhhvCg8J5twBCZw7IIEfXjKEI5W1fPSlExyr8kp4Z8sBAHp3j2hqdUwYlEhiF56iazflGWPMGfqqtJKV+cV8mF/Ch/mlHDnmPPwps3ds03jH+P6dY4qu36fOtjcLC2OMP9Q3KBsLj7Aqv4SVecWs3e1M0Q0LCWJceg/OH5TEBRkdd4quhYUxxvhBZU0dn+48yKo8p8tqW5Gzgq7nFN0JgxJJ69HNz5U6/D5mYYwxgahbWAiThyQzecjxKbofflni3N+Rd3yKbv/EqKbxjvMGJnT4R81ay8IYY9qJqpJ34KgbHMV8uvMglTX1BAcJo9O6c36G02XVnlN0rRvKGGM6uJq6Bj7/6tDXpuhGhTmzsc7PSOSCjESfPrvDwsIYYzqZI5W1fLzD7bLKL2F3aSXgTNGdMCixabzDm1N0LSyMMaaT23OwsmktK88pusPcVXQnDEpkfHo8kWFtn6JrYWGMMV1IfYOyae+RpoHytbsPUVPfQFhIEBdn9uSP145p03VtNpQxxnQhwUHCqLQ4RqXFcefkQVTWOKvorsorISzE94PhFhbGGNMJdQsLcR8jm9wu7xeYyycaY4w5IxYWxhhjWmRhYYwxpkUWFsYYY1pkYWGMMaZFFhbGGGNaZGFhjDGmRRYWxhhjWtRllvsQkWJg91lcIhEo8VI5nUWgfeZA+7xgnzlQnM1n7qeqSS0d1GXC4myJyJrWrI/SlQTaZw60zwv2mQNFe3xm64YyxhjTIgsLY4wxLbKwOG6Bvwvwg0D7zIH2ecE+c6Dw+We2MQtjjDEtspaFMcaYFllYGGOMaVHAh4WITBeRbSKSLyL3+7seXxORp0XkgIhs9Hct7UVE+ojIchHZLCKbROT7/q7J10QkQkQ+E5H17mf+L3/X1B5EJFhE1onI6/6upb2IyC4R+UJEckXEZ8+WDugxCxEJBrYDFwEFwGpgrqpu9mthPiQiFwJHgedVdYS/62kPItIb6K2qn4tIDLAWmN3F/zsLEKWqR0UkFFgFfF9VP/FzaT4lIvcBOUCsqn7T3/W0BxHZBeSoqk9vRAz0lsV4IF9Vd6hqDbAQmOXnmnxKVT8ADvq7jvakqvtU9XP353JgC5Dq36p8Sx1H3Zeh7leX/stQRNKAy4An/V1LVxToYZEK7PF4XUAX/yUS6EQkHcgGPvVvJb7ndsnkAgeAt1W1q3/m+cC/Aw3+LqSdKfCWiKwVkXm+epNADwsTQEQkGngVuFdVy/xdj6+par2qZgFpwHgR6bLdjiLyTeCAqq71dy1+cL6qjgEuBe50u5q9LtDDohDo4/E6zd1muhi33/5V4K+q+g9/19OeVPUwsByY7u9afGgCMNPtv18ITBGRv/i3pPahqoXu9wPAIpzuda8L9LBYDWSISH8RCQOuAZb4uSbjZe5g71PAFlV9yN/1tAcRSRKROPfnSJxJHFv9W5XvqOqPVTVNVdNx/n/8nqpe7+eyfE5EotxJG4hIFHAx4JOZjgEdFqpaB9wFLMMZ9HxZVTf5tyrfEpGXgI+BISJSICK3+LumdjAB+A7OX5u57tcMfxflY72B5SKyAeePordVNWCmkwaQnsAqEVkPfAa8oar/8sUbBfTUWWOMMa0T0C0LY4wxrWNhYYwxpkUWFsYYY1pkYWGMMaZFFhbGGGNaZGFhzBkQkXqP6be53lypWETSA2k1YNO5hPi7AGM6mWPuEhrGBBRrWRjjBe4zBX7jPlfgMxEZ5G5PF5H3RGSDiLwrIn3d7T1FZJH7vIn1IvIN91LBIvKE+wyKt9y7r43xOwsLY85M5EndUFd77DuiqiOBP+KsgArwKPCcqo4C/go84m5/BHhfVUcDY4DGlQMygMdUdThwGPiWjz+PMa1id3AbcwZE5KiqRjezfRcwRVV3uIsW7lfVBBEpwXnwUq27fZ+qJopIMZCmqtUe10jHWZYjw339IyBUVX/p+09mzOlZy8IY79FT/Hwmqj1+rsfGFU0HYWFhjPdc7fH9Y/fnj3BWQQW4Dljp/vwucAc0PaSoe3sVaUxb2F8txpyZSPfpc43+paqN02d7uKu8VgNz3W13A8+IyP8DioGb3O3fBxa4q/7W4wTHPp9Xb0wb2ZiFMV7gjlnkqGqJv2sxxhesG8oYY0yLrGVhjDGmRdayMMYY0yILC2OMMS2ysDDGGNMiCwtjjDEtsrAwxhjTov8PU/WxcfK7S7IAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluation = model.evaluate(x = X_test, y = y_test, verbose=1) \n",
    "loss = evaluation[0]\n",
    "acc = evaluation[1]\n",
    "print('LOSS ON TEST DATA='+str(loss))\n",
    "print('ACCURACY ON TEST DATA='+str(acc))\n",
    "\n",
    "\n",
    "# Plot training & validation loss values\n",
    "plt.plot(history.history['loss'], label='Train')\n",
    "plt.plot(history.history['val_loss'], label='Validation')\n",
    "plt.title('Model loss')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Validation'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99       484\n",
      "           1       0.94      0.94      0.94        95\n",
      "\n",
      "   micro avg       0.98      0.98      0.98       579\n",
      "   macro avg       0.96      0.96      0.96       579\n",
      "weighted avg       0.98      0.98      0.98       579\n",
      "\n",
      "Accuracy: 0.9792746113989638\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAF4VJREFUeJzt3X20XXV95/H3xwREBYMasJoEgjxUqU9oCjiupcyIFBkNLusDGdHiUNJaaXW0ts6qo5RWrTrasQWrVCiKCoJjXWmNxSc0oxVNHJACCo3IQwIOIBBEMDx954+9Lzlekn1Prnffc254v9a6i/3wO3t/94+b87l7/87ZO1WFJEnb8rBRFyBJGm8GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBoTklyWVJDpuizV5J7kgyb5bK6l2Sq5Mc3k6flOSTo65JDx0GhWZE+0Z2V/sG/f+SnJlk15neT1X9RlV9fYo211bVrlV130zvv32Tvqc9ztuS/GuS58z0fqRxYlBoJr2kqnYFngUsA94+uUEac/337jPtcS4ELgDOG3E9My7J/FHXoPEx1//BagxV1Ubgi8BTAZJ8Pcm7knwLuBN4UpIFSU5PckOSjUn+cvBSUZITkvwgyc+SXJ7kWe3ywUswBydZl+T29izmg+3ypUlq4s0uyROTrEpyS5L1SU4Y2M9JSc5N8ol2X5clWTbkcd4LfApYlGSPgW2+OMnFA2ccTx9YtyTJ55LclOSnSU5pl++b5GvtspuTfCrJ7tPp/yRHt/u/PcmPkhw5ue8Gjv2Tk/rs+CTXAl9L8sUkJ07a9veTvKydfnKSL7f9ekWSV06nXo0/g0IzLskS4CjgooHFrwFWArsB1wBnAvcC+wEHAUcAv9u+/hXAScBrgUcDy4GfbmVXHwI+VFWPBvYFzt1GSecAG4AnAi8H3p3kPw2sX9622R1YBZwy5HHu3Nb4U+DWdtlBwBnA7wGPAz4KrEry8DYI/7k9/qXAona/AAHe09b4FGBJ2wfbJcnBwCeAt7bH8zzg6u3YxPPb/f8WcDawYmDbBwJ7A19I8ijgy8CngT2BY4APt220gzEoNJM+n+Q24JvAN4B3D6w7s6oua/8KfyxNkLypqn5eVTcCf03zZgNNYLyvqtZWY31VXbOV/d0D7JdkYVXdUVUXTm7QhtZzgT+tql9U1cXAx2je4Cd8s6pWt2MaZwHPmOI4X9ke513ACcDL2+OCJgw/WlXfqar7qurjwGbgUOBgmiB4a3vcv6iqbwK0x/jlqtpcVTcBH6R5095exwNntNu6v6o2VtUPt+P1J7W13QX8I/DMJHu3614NfK6qNgMvBq6uqn+oqnur6iLgfwOvmEbNGnMGhWbSS6tq96rau6r+oH2zmXDdwPTewE7ADe3lmdto/vLes12/BPjREPs7HjgA+GGStUlevJU2TwRuqaqfDSy7huav+Qk/GZi+E9glyfwkr24Hre9I8sWBNudW1e7A44FLgWdPOra3TBxXe2xL2jqWANcMhMoDkjw+yTntZbjbgU/SjIFsr2H7blse+P/U9tkX2BLgK2gutUFznIdMOs5XA7/2K+xbY8oBK82WwdsUX0fzV/bCrb1ptuv3nXKDVf8OrGgHx18GfDbJ4yY1ux54bJLdBsJiL2DjENv/FFveGLe2/uYkK4F1ST5dVTe0tb+rqt41uX376ai9kszfynG/m6aPnlZVtyR5KUNeApukq+9+DjxyYH5rb+qTbyd9NvDOJGuAXWgG7yf2842qeuE0atQc4xmFZl37hvol4ANJHp3kYe1g7sSllo8Bf5zk2c2HpLLfwOWPByQ5NskeVXU/cFu7+P5J+7oO+FfgPUl2aQeWj6f5i30mjuUK4HzgT9pFfw/8fpJD2tofleQ/J9kN+C5wA/BX7fJdkjy3fd1uwB3ApiSLaMYYpuN04HVJXtD266IkT27XXQwck2SndsD+5UNsbzXN2cPJNJ/2mujffwYOSPKadns7JfnNJE+ZZt0aYwaFRuW1wM7A5TQDwZ8FngBQVecB76IZKP0Z8HmacY3JjgQuS3IHzcD2MZMud01YQTN4fD3Ndfd3VtVXZvBY3g+sTLJnVa2jGbc4pT2u9cBxAO0YyEtoBvCvpRlgf1W7jT+n+VjxJprLPZ+bTiFV9V3gdTRjPptoxoomQvZ/0Jxt3Nru79NDbG9zW8vhg+3bs7MjaC5LXU9z+e69wMOnU7fGW3xwkSSpi2cUkqROBoUkqZNBIUnqZFBIkjrNue9RLFy4sJYuXTrqMiRpTvne9753c1XtMXXLB5tzQbF06VLWrVs36jIkaU5JsrXb4AzFS0+SpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqVNvQZHkjCQ3Jrl0G+uT5G/SPMP4krTPRJYkjZc+zyjOpLkN9La8CNi//VkJ/F2PtUiSpqm3oKiqNcAtHU2OBj7RPhP5QmD3JE+Yart33z1TFUqShjHKMYpF/PJzlDfwy88xfkCSlUnWJVl3ww23zkpxkqTGnBjMrqrTqmpZVS1bsOAxoy5Hkh5SRhkUG4ElA/OLGeKB95Kk2TXKoFgFvLb99NOhwKaqumGE9UiStqK3u8cmORs4DFiYZAPwTmAngKr6CLAaOIrm4fN30jwQXpI0ZnoLiqpaMcX6At7Q1/4lSTNjTgxmS5JGx6CQJHUyKCRJnQwKSVIng0KS1MmgkCR16u3jsX25/3648spRVyFJ/XnsY2HhwlFXscWcCwqANWtGXYEk9WPz5iYoVnR+E212zbmgmD8fDjpo1FVIUj+uuQZu6XpAwwg4RiFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTr0GRZIjk1yRZH2St21l/V5JLkhyUZJLkhzVZz2SpO3XW1AkmQecCrwIOBBYkeTASc3eDpxbVQcBxwAf7qseSdL09HlGcTCwvqquqqq7gXOAoye1KeDR7fQC4Poe65EkTcP8Hre9CLhuYH4DcMikNicBX0ryh8CjgMO3tqEkK4GVAHvuudeMFypJ2rZRD2avAM6sqsXAUcBZSR5UU1WdVlXLqmrZggV7zHqRkvRQ1mdQbASWDMwvbpcNOh44F6Cqvg3sAizssSZJ0nbqMyjWAvsn2SfJzjSD1asmtbkWeAFAkqfQBMVNPdYkSdpOvQVFVd0LnAicD/yA5tNNlyU5OcnyttlbgBOSfB84GziuqqqvmiRJ26/PwWyqajWwetKydwxMXw48t88aJEm/mlEPZkuSxpxBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6zR+2YZJFwN6Dr6mqNX0UJUkaH0MFRZL3Aq8CLgfuaxcX0BkUSY4EPgTMAz5WVX+1lTavBE5qt/f9qvovwxYvSerfsGcULwV+vao2D7vhJPOAU4EXAhuAtUlWVdXlA232B/478NyqujXJnsOXLkmaDcOOUVwF7LSd2z4YWF9VV1XV3cA5wNGT2pwAnFpVtwJU1Y3buQ9JUs+GPaO4E7g4yVeBB84qquqPOl6zCLhuYH4DcMikNgcAJPkWzeWpk6rqX4asSZI0C4YNilXtTx/73x84DFgMrEnytKq6bbBRkpXASoA999yrhzIkSdsyVFBU1ceT7Ex7BgBcUVX3TPGyjcCSgfnF7bJBG4DvtNv6cZIraYJj7aT9nwacBnDAActqmJolSTNjqDGKJIcB/04zOP1h4Mokz5viZWuB/ZPs04bMMTz4rOTzNGcTJFlIE0RXDVu8JKl/w156+gBwRFVdAZDkAOBs4NnbekFV3ZvkROB8mvGHM6rqsiQnA+uqalW77ogkEx+7fWtV/XT6hyNJmmnDBsVOEyEBUFVXJpnyU1BVtRpYPWnZOwamC3hz+yNJGkPDBsW6JB8DPtnOvxpY109JkqRxMmxQvB54AzDxcdj/QzNWIUnawQ37qafNwAfbH0nSQ0hnUCQ5t6pemeTfaO7F9Euq6um9VSZJGgtTnVG8sf3vi/suRJI0njq/R1FVN7STNwPXVdU1wMOBZwDX91ybJGkMDHtTwDXALu0zKb4EvAY4s6+iJEnjY9igSFXdCbwM+HBVvQL4jf7KkiSNi6GDIslzaL4/8YV22bx+SpIkjZNhg+JNNA8Y+sf2NhxPAi7oryxJ0rgY9nsU3wC+MTB/FVu+fCdJ2oFN9T2K/1VVb0ryT2z9exTLe6tMkjQWpjqjOKv97//suxBJ0njqDIqq+l47uQ64q6ruB0gyj+b7FJKkHdywg9lfBR45MP8I4CszX44kadwMGxS7VNUdEzPt9CM72kuSdhDDBsXPkzxrYibJs4G7+ilJkjROhn0exZuA85JcDwT4NeBVvVUlSRobw36PYm2SJwO/3i66oqru6a8sSdK4GOrSU5JHAn8KvLGqLgWWJvHW45L0EDDsGMU/AHcDz2nnNwJ/2UtFkqSxMmxQ7FtV7wPuAWjvJJveqpIkjY1hg+LuJI+gvY1Hkn2Bzb1VJUkaG8N+6umdwL8AS5J8CngucFxfRUmSxseUQZEkwA9pHlp0KM0lpzdW1c091yZJGgNTBkVVVZLVVfU0tjy0SJL0EDHsGMX/TfKbvVYiSRpLw45RHAIcm+Rq4Oc0l5+qqp7eV2GSpPEwbFD8Vq9VSJLG1lRPuNsF+H1gP+DfgNOr6t7ZKEySNB6mGqP4OLCMJiReBHyg94okSWNlqktPB7afdiLJ6cB3+y9JkjROpjqjeOAOsV5ykqSHpqmC4hlJbm9/fgY8fWI6ye1TbTzJkUmuSLI+yds62v12kkqybHsPQJLUr85LT1U1b7obTjIPOBV4IbABWJtkVVVdPqndbsAbge9Md1+SpP4M+4W76TgYWF9VV1XV3cA5wNFbafcXwHuBX/RYiyRpmvoMikXAdQPzG9plD2ifw72kqjpvDZJkZZJ1SdZt2nTTzFcqSdqmPoOiU5KHAR8E3jJV26o6raqWVdWyBQv26L84SdID+gyKjcCSgfnF7bIJuwFPBb7e3hrkUGCVA9qSNF76DIq1wP5J9kmyM3AMsGpiZVVtqqqFVbW0qpYCFwLLq2pdjzVJkrZTb0HRfu/iROB84AfAuVV1WZKTkyzva7+SpJk17E0Bp6WqVgOrJy17xzbaHtZnLZKk6RnZYLYkaW4wKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktSp16BIcmSSK5KsT/K2rax/c5LLk1yS5KtJ9u6zHknS9ustKJLMA04FXgQcCKxIcuCkZhcBy6rq6cBngff1VY8kaXr6PKM4GFhfVVdV1d3AOcDRgw2q6oKqurOdvRBY3GM9kqRp6DMoFgHXDcxvaJdty/HAF7e2IsnKJOuSrNu06aYZLFGSNJWxGMxOciywDHj/1tZX1WlVtayqli1YsMfsFidJD3Hze9z2RmDJwPzidtkvSXI48GfA86tqc4/1SJKmoc8zirXA/kn2SbIzcAywarBBkoOAjwLLq+rGHmuRJE1Tb0FRVfcCJwLnAz8Azq2qy5KcnGR52+z9wK7AeUkuTrJqG5uTJI1In5eeqKrVwOpJy94xMH14n/uXJP3qxmIwW5I0vno9o5Akbb/Nm+HKK0ddxRYGhSSNkQUL4Cc/gTVrZnrLuz1quq80KCRpjOy+OxxySB9bnjdvuq90jEKS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktSp16BIcmSSK5KsT/K2rax/eJLPtOu/k2Rpn/VIkrZfb0GRZB5wKvAi4EBgRZIDJzU7Hri1qvYD/hp4b1/1SJKmp88zioOB9VV1VVXdDZwDHD2pzdHAx9vpzwIvSJIea5Ikbaf5PW57EXDdwPwG4JBttamqe5NsAh4H3DzYKMlKYGU7d8+yZY+5upeK55zNC+Dhm0ZdxXiwL7awL7awL7a4ffF0X9lnUMyYqjoNOA0gybqqW5eNuKSx0PTFnfYF9sUg+2IL+2KLJOum+9o+Lz1tBJYMzC9ul221TZL5wALgpz3WJEnaTn0GxVpg/yT7JNkZOAZYNanNKuB32umXA1+rquqxJknSdurt0lM75nAicD4wDzijqi5LcjKwrqpWAacDZyVZD9xCEyZTOa2vmucg+2IL+2IL+2IL+2KLafdF/ANektTFb2ZLkjoZFJKkTmMbFN7+Y4sh+uLNSS5PckmSrybZexR1zoap+mKg3W8nqSQ77Ecjh+mLJK9sfzcuS/Lp2a5xtgzxb2SvJBckuaj9d3LUKOrsW5IzktyY5NJtrE+Sv2n76ZIkzxpqw1U1dj80g98/Ap4E7Ax8HzhwUps/AD7STh8DfGbUdY+wL/4j8Mh2+vUP5b5o2+0GrAEuBJaNuu4R/l7sD1wEPKad33PUdY+wL04DXt9OHwhcPeq6e+qL5wHPAi7dxvqjgC8CAQ4FvjPMdsf1jMLbf2wxZV9U1QVVdWc7eyHNd1Z2RMP8XgD8Bc19w34xm8XNsmH64gTg1Kq6FaCqbpzlGmfLMH1RwKPb6QXA9bNY36ypqjU0nyDdlqOBT1TjQmD3JE+YarvjGhRbu/3Hom21qap7gYnbf+xohumLQcfT/MWwI5qyL9pT6SVV9YXZLGwEhvm9OAA4IMm3klyY5MhZq252DdMXJwHHJtkArAb+cHZKGzvb+34CzJFbeGg4SY4FlgHPH3Uto5DkYcAHgeNGXMq4mE9z+ekwmrPMNUmeVlW3jbSq0VgBnFlVH0jyHJrvbz21qu4fdWFzwbieUXj7jy2G6QuSHA78GbC8qjbPUm2zbaq+2A14KvD1JFfTXINdtYMOaA/ze7EBWFVV91TVj4EraYJjRzNMXxwPnAtQVd8GdgEWzkp142Wo95PJxjUovP3HFlP2RZKDgI/ShMSOeh0apuiLqtpUVQuramlVLaUZr1leVdO+GdoYG+bfyOdpziZIspDmUtRVs1nkLBmmL64FXgCQ5Ck0QXHTrFY5HlYBr20//XQosKmqbpjqRWN56an6u/3HnDNkX7wf2BU4rx3Pv7aqlo+s6J4M2RcPCUP2xfnAEUkuB+4D3lpVO9xZ95B98Rbg75P8N5qB7eN2xD8sk5xN88fBwnY85p3ATgBV9RGa8ZmjgPXAncDrhtruDthXkqQZNK6XniRJY8KgkCR1MigkSZ0MCklSJ4NCktTJoJAmSXJfkouTXJrkn5LsPsPbPy7JKe30SUn+eCa3L800g0J6sLuq6plV9VSa7+i8YdQFSaNkUEjdvs3ATdOSvDXJ2vZe/n8+sPy17bLvJzmrXfaS9lkpFyX5SpLHj6B+6Vc2lt/MlsZBknk0t304vZ0/guZeSQfT3M9/VZLn0dxj7O3Af6iqm5M8tt3EN4FDq6qS/C7wJzTfEJbmFINCerBHJLmY5kziB8CX2+VHtD8XtfO70gTHM4DzqupmgKqaeB7AYuAz7f3+dwZ+PDvlSzPLS0/Sg91VVc8E9qY5c5gYowjwnnb84plVtV9Vnd6xnb8FTqmqpwG/R3MjOmnOMSikbWifGvhHwFvaW9mfD/zXJLsCJFmUZE/ga8ArkjyuXT5x6WkBW27h/DtIc5SXnqQOVXVRkkuAFVV1VnuL6m+3d+m9Azi2vVPpu4BvJLmP5tLUcTRPVTsvya00YbLPKI5B+lV591hJUicvPUmSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKnT/weOSANgjIikoQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluate(model, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Self attention layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "from keras import initializers, regularizers, constraints\n",
    "from keras.layers.core import Layer\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "def dot_product(x, kernel):\n",
    "    \"\"\"\n",
    "    Wrapper for dot product operation, in order to be compatible with both\n",
    "    Theano and Tensorflow\n",
    "    Args:\n",
    "        x (): input\n",
    "        kernel (): weights\n",
    "    Returns:\n",
    "    \"\"\"\n",
    "    if K.backend() == 'tensorflow':\n",
    "        return K.squeeze(K.dot(x, K.expand_dims(kernel)), axis=-1)\n",
    "    else:\n",
    "        return K.dot(x, kernel)\n",
    "\n",
    "\n",
    "class SimpleAttention(Layer):\n",
    "    def __init__(self,\n",
    "                 kernel_regularizer=None, bias_regularizer=None,\n",
    "                 W_constraint=None, b_constraint=None,\n",
    "                 bias=True,\n",
    "                 return_attention=False,\n",
    "                 **kwargs):\n",
    "        \n",
    "        self.supports_masking = True\n",
    "        self.init = initializers.get('glorot_uniform')\n",
    "\n",
    "        self.W_regularizer = regularizers.get(kernel_regularizer)\n",
    "        self.b_regularizer = regularizers.get(bias_regularizer)\n",
    "\n",
    "        self.W_constraint = constraints.get(W_constraint)\n",
    "        self.b_constraint = constraints.get(b_constraint)\n",
    "\n",
    "        self.bias = bias\n",
    "        self.return_attention = return_attention\n",
    "        super(SimpleAttention, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        assert len(input_shape) == 3\n",
    "\n",
    "        self.W = self.add_weight((input_shape[-1],),\n",
    "                                 initializer=self.init,\n",
    "                                 name='{}_W'.format(self.name),\n",
    "                                 regularizer=self.W_regularizer,\n",
    "                                 constraint=self.W_constraint)\n",
    "        if self.bias:\n",
    "            self.b = self.add_weight((1,),\n",
    "                                     initializer='zero',\n",
    "                                     name='{}_b'.format(self.name),\n",
    "                                     regularizer=self.b_regularizer,\n",
    "                                     constraint=self.b_constraint)\n",
    "        else:\n",
    "            self.b = None\n",
    "\n",
    "        self.built = True\n",
    "\n",
    "    def compute_mask(self, inputs, mask=None):\n",
    "        # do not pass the mask to the next layers\n",
    "        if self.return_attention:\n",
    "            return [None, None]\n",
    "        return None\n",
    "\n",
    "    def call(self, x, mask=None):\n",
    "        \n",
    "        # eij = Wx + b\n",
    "        eij = dot_product(x, self.W)\n",
    "\n",
    "        if self.bias:\n",
    "            eij += self.b\n",
    "\n",
    "        # Apply mask\n",
    "        if mask is not None:\n",
    "            eij *= K.cast(mask, K.floatx())\n",
    "\n",
    "        # a = softmax(eij)\n",
    "        a = K.expand_dims(K.softmax(eij, axis=-1))\n",
    "        weighted_input = x * a\n",
    "        result = K.sum(weighted_input, axis=1)\n",
    "\n",
    "        if self.return_attention:\n",
    "            return [result, a]\n",
    "        return result\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        if self.return_attention:\n",
    "            return [(input_shape[0], input_shape[-1]),\n",
    "                    (input_shape[0], input_shape[1])]\n",
    "        else:\n",
    "            return input_shape[0], input_shape[-1]\n",
    "\n",
    "\n",
    "class DeepAttention(Layer):\n",
    "    def __init__(self,\n",
    "                 kernel_regularizer=None, u_regularizer=None, bias_regularizer=None,\n",
    "                 W_constraint=None, u_constraint=None, b_constraint=None,\n",
    "                 bias=True,\n",
    "                 return_attention=False,\n",
    "                 **kwargs):\n",
    "\n",
    "        self.supports_masking = True\n",
    "        self.init = initializers.get('glorot_uniform')\n",
    "\n",
    "        self.W_regularizer = regularizers.get(kernel_regularizer)\n",
    "        self.u_regularizer = regularizers.get(u_regularizer)\n",
    "        self.b_regularizer = regularizers.get(bias_regularizer)\n",
    "\n",
    "        self.W_constraint = constraints.get(W_constraint)\n",
    "        self.u_constraint = constraints.get(u_constraint)\n",
    "        self.b_constraint = constraints.get(b_constraint)\n",
    "\n",
    "        self.bias = bias\n",
    "        self.return_attention = return_attention\n",
    "        super(DeepAttention, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        assert len(input_shape) == 3\n",
    "\n",
    "        self.W = self.add_weight((input_shape[-1], input_shape[-1],),\n",
    "                                 initializer=self.init,\n",
    "                                 name='{}_W'.format(self.name),\n",
    "                                 regularizer=self.W_regularizer,\n",
    "                                 constraint=self.W_constraint)\n",
    "        if self.bias:\n",
    "            self.b = self.add_weight((input_shape[-1],),\n",
    "                                     initializer='zero',\n",
    "                                     name='{}_b'.format(self.name),\n",
    "                                     regularizer=self.b_regularizer,\n",
    "                                     constraint=self.b_constraint)\n",
    "        else:\n",
    "            self.b = None\n",
    "\n",
    "        self.u = self.add_weight((input_shape[-1],),\n",
    "                                 initializer=self.init,\n",
    "                                 name='{}_u'.format(self.name),\n",
    "                                 regularizer=self.u_regularizer,\n",
    "                                 constraint=self.u_constraint)\n",
    "\n",
    "        super(DeepAttention, self).build(input_shape)\n",
    "\n",
    "    def compute_mask(self, inputs, mask=None):\n",
    "        # do not pass the mask to the next layers\n",
    "        if self.return_attention:\n",
    "            return [None, None]\n",
    "        return None\n",
    "\n",
    "    def call(self, x, mask=None):\n",
    "        # uit = tanh(Wx + b)\n",
    "        uit = dot_product(x, self.W)\n",
    "\n",
    "        if self.bias:\n",
    "            uit += self.b\n",
    "\n",
    "        uit = K.tanh(uit)\n",
    "\n",
    "        # ait = softmax(Ueij)\n",
    "        eij = dot_product(uit, self.u)\n",
    "\n",
    "        # Apply mask\n",
    "        if mask is not None:\n",
    "            eij *= K.cast(mask, K.floatx())\n",
    "\n",
    "        a = K.expand_dims(K.softmax(eij, axis=-1))\n",
    "        \n",
    "        weighted_input = x * a\n",
    "        result = K.sum(weighted_input, axis=1)\n",
    "\n",
    "        if self.return_attention:\n",
    "            return [result, a]\n",
    "        return result\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        if self.return_attention:\n",
    "            return [(input_shape[0], input_shape[-1]),\n",
    "                    (input_shape[0], input_shape[1])]\n",
    "        else:\n",
    "            return input_shape[0], input_shape[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMBEDDING_DIM = 300\n",
    "embedding_matrix = np.zeros((len(word_index) + 1, EMBEDDING_DIM))\n",
    "for word, i in word_index.items():\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Embedding\n",
    "\n",
    "embedding_layer = Embedding(len(word_index) + 1,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(data, y, test_size=0.2, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_up(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 150)               0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, 150, 300)          18288300  \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 150, 300)          0         \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 150, 200)          320800    \n",
      "_________________________________________________________________\n",
      "bidirectional_2 (Bidirection (None, 150, 200)          240800    \n",
      "_________________________________________________________________\n",
      "deep_attention_1 (DeepAttent (None, 200)               40400     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 201       \n",
      "=================================================================\n",
      "Total params: 18,890,501\n",
      "Trainable params: 602,201\n",
      "Non-trainable params: 18,288,300\n",
      "_________________________________________________________________\n",
      "Train on 2082 samples, validate on 232 samples\n",
      "Epoch 1/20\n",
      "2082/2082 [==============================] - 18s 9ms/step - loss: 0.3102 - acc: 0.8631 - val_loss: 0.1734 - val_acc: 0.9397\n",
      "Epoch 2/20\n",
      "2082/2082 [==============================] - 15s 7ms/step - loss: 0.0750 - acc: 0.9774 - val_loss: 0.0859 - val_acc: 0.9828\n",
      "Epoch 3/20\n",
      "2082/2082 [==============================] - 15s 7ms/step - loss: 0.0503 - acc: 0.9846 - val_loss: 0.0761 - val_acc: 0.9784\n",
      "Epoch 4/20\n",
      "2082/2082 [==============================] - 15s 7ms/step - loss: 0.0340 - acc: 0.9909 - val_loss: 0.0606 - val_acc: 0.9828\n",
      "Epoch 5/20\n",
      "2082/2082 [==============================] - 15s 7ms/step - loss: 0.0274 - acc: 0.9923 - val_loss: 0.0738 - val_acc: 0.9871\n",
      "Epoch 6/20\n",
      "2082/2082 [==============================] - 15s 7ms/step - loss: 0.0516 - acc: 0.9837 - val_loss: 0.0499 - val_acc: 0.9957\n",
      "Epoch 7/20\n",
      "2082/2082 [==============================] - 16s 7ms/step - loss: 0.0299 - acc: 0.9894 - val_loss: 0.0871 - val_acc: 0.9784\n",
      "Epoch 8/20\n",
      "2082/2082 [==============================] - 15s 7ms/step - loss: 0.0171 - acc: 0.9957 - val_loss: 0.0485 - val_acc: 0.9914\n",
      "Epoch 9/20\n",
      "2082/2082 [==============================] - 15s 7ms/step - loss: 0.0093 - acc: 0.9981 - val_loss: 0.0478 - val_acc: 0.9914\n",
      "Epoch 10/20\n",
      "2082/2082 [==============================] - 15s 7ms/step - loss: 0.0055 - acc: 0.9990 - val_loss: 0.0441 - val_acc: 0.9957\n",
      "Epoch 11/20\n",
      "2082/2082 [==============================] - 15s 7ms/step - loss: 0.0080 - acc: 0.9966 - val_loss: 0.0719 - val_acc: 0.9828\n",
      "Epoch 12/20\n",
      "2082/2082 [==============================] - 15s 7ms/step - loss: 0.0057 - acc: 0.9981 - val_loss: 0.0720 - val_acc: 0.9871\n",
      "Restoring model weights from the end of the best epoch\n",
      "Epoch 00012: early stopping\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Model\n",
    "\n",
    "inputs =  Input(shape=(150, ), dtype='int32')\n",
    "embeddings = embedding_layer(inputs)\n",
    "dropped_embeddings = Dropout(0.0)(embeddings)\n",
    "lstms_1 = Bidirectional(LSTM(100, return_sequences=True))(dropped_embeddings)\n",
    "lstms_2 = Bidirectional(LSTM(100, return_sequences=True))(lstms_1)\n",
    "document_encoding =  DeepAttention()(lstms_2)\n",
    "outputs = Dense(1, activation='sigmoid')(document_encoding)\n",
    "\n",
    "# Wrap model\n",
    "model = Model(inputs=inputs, outputs=outputs)\n",
    "model.summary()\n",
    "\n",
    "es = EarlyStopping(monitor='val_loss', patience=2, restore_best_weights=True, verbose=1)\n",
    "\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "history = model.fit(X_train, y_train, epochs=20, batch_size=64, verbose=1, validation_split=0.1, callbacks=[es])\n",
    "\n",
    "y_pred = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "579/579 [==============================] - 3s 5ms/step\n",
      "LOSS ON TEST DATA=0.03623503053197231\n",
      "ACCURACY ON TEST DATA=0.9930915371329879\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8lNXZ//HPlclKFpYkkEBYwmYCigIRVBRFUMGNapFCtY9aldZq7VNbW+1ma+tTay3VWp/+qq221oUHsVqqUETcFxRQRCFBILIEAllYsq9z/f64J2EICZkkM5lk5nq/XvNi5r7PPXOCON+c5T5HVBVjjDHmRCKCXQFjjDE9n4WFMcaYdllYGGOMaZeFhTHGmHZZWBhjjGmXhYUxxph2WVgY0wUiMkJEVEQifSh7nYi809X3MSYYLCxM2BCRnSJSJyIpLY5/7PmiHhGcmhnT81lYmHDzBbCw6YWInAL0CV51jOkdLCxMuPkH8F9er68FnvQuICJ9ReRJESkWkV0i8hMRifCcc4nIAyJSIiL5wCWtXPtXESkUkb0i8isRcXW0kiIyWESWi8hBEdkuIjd5nZsiIutFpExEDojIYs/xWBF5SkRKReSwiKwTkUEd/WxjWmNhYcLNWiBJRLI9X+ILgKdalHkY6AuMBM7FCZfrPeduAi4FJgI5wLwW1/4NaABGe8pcCNzYiXouAQqAwZ7P+B8ROd9z7iHgIVVNAkYBSz3Hr/XUeyiQDHwTqO7EZxtzHAsLE46aWhcXALnA3qYTXgFyl6qWq+pO4HfA1zxF5gMPquoeVT0I/Nrr2kHAxcB/q2qlqhYBv/e8n89EZCgwDfihqtao6kbgLxxtEdUDo0UkRVUrVHWt1/FkYLSqNqrqBlUt68hnG9MWCwsTjv4BfBW4jhZdUEAKEAXs8jq2CxjieT4Y2NPiXJPhnmsLPd1Ah4E/AwM7WL/BwEFVLW+jDjcAY4E8T1fTpV4/1ypgiYjsE5H7RSSqg59tTKssLEzYUdVdOAPdFwP/bHG6BOc39OFex4ZxtPVRiNPN432uyR6gFkhR1X6eR5Kqju9gFfcBA0QksbU6qOo2VV2IE0K/AZaJSLyq1qvqL1R1HHAWTnfZf2GMH1hYmHB1A3C+qlZ6H1TVRpwxgHtFJFFEhgO3c3RcYylwm4hkiEh/4E6vawuBV4DfiUiSiESIyCgRObcjFVPVPcB7wK89g9YTPPV9CkBErhGRVFV1A4c9l7lFZIaInOLpSivDCT13Rz7bmLZYWJiwpKo7VHV9G6e/DVQC+cA7wDPA455zj+F09XwCfMTxLZP/AqKBLcAhYBmQ3okqLgRG4LQyXgDuVtVXPedmA5tFpAJnsHuBqlYDaZ7PK8MZi3kTp2vKmC4T2/zIGGNMe6xlYYwxpl0WFsYYY9plYWGMMaZdFhbGGGPaFTLLIaekpOiIESOCXQ1jjOlVNmzYUKKqqe2VC5mwGDFiBOvXtzUT0hhjTGtEZFf7pawbyhhjjA8sLIwxxrTLwsIYY0y7QmbMojX19fUUFBRQU1MT7KqElNjYWDIyMoiKsgVNjQkXIR0WBQUFJCYmMmLECEQk2NUJCapKaWkpBQUFZGZmBrs6xphuEtLdUDU1NSQnJ1tQ+JGIkJycbK01Y8JMSIcFYEERAPZ3akz4CfmwaE9Do5sDZTVU1TUEuyrGGNNjhX1YiMCBshoqavwfFqWlpZx22mmcdtpppKWlMWTIkObXdXV1Pr3H9ddfz9atW/1eN2OM6YiQHuD2hSsigujICKrrG/3+3snJyWzcuBGAn//85yQkJPD973//mDKqiqoSEdF6bj/xxBN+r5cxxnRU2LcsAGIjXdTUd9/uk9u3b2fcuHFcffXVjB8/nsLCQhYtWkROTg7jx4/nnnvuaS579tlns3HjRhoaGujXrx933nknp556KmeeeSZFRUXdVmdjTHgLm5bFL/69mS37ylo9V9fopr7BTXxMx/46xg1O4u7LxneqPnl5eTz55JPk5OQAcN999zFgwAAaGhqYMWMG8+bNY9y4ccdcc+TIEc4991zuu+8+br/9dh5//HHuvPPO1t7eGGP8KqAtCxGZLSJbRWS7iBz3rSYi3xSRT0Vko4i8IyLjvM7d5bluq4hcFMh6ujyze9zduMXsqFGjmoMC4Nlnn2XSpElMmjSJ3NxctmzZctw1cXFxzJkzB4DJkyezc+fO7qquMSbMBaxlISIu4BHgAqAAWCciy1XV+1vwGVX9f57ylwOLgdme0FgAjAcGA6+KyFhV7fTAwolaALX1jWw9UE5G/z4MiI/u7Ed0SHx8fPPzbdu28dBDD/Hhhx/Sr18/rrnmmlbvY4iOPlo3l8tFQ4PN4DLGdI9AtiymANtVNV9V64AlwFzvAqrq3S8UDzT9aj8XWKKqtar6BbDd834BER0ZQYQINQEY5PZFWVkZiYmJJCUlUVhYyKpVq4JSD2OMaUsgxyyGAHu8XhcAU1sWEpFbgNuBaOB8r2vXtrh2SCvXLgIWAQwbNqzTFRURYqNcAZkR5YtJkyYxbtw4srKyGD58ONOmTQtKPYwxpi2iAeqnF5F5wGxVvdHz+mvAVFW9tY3yXwUuUtVrReSPwFpVfcpz7q/ASlVd1tbn5eTkaMvNj3Jzc8nOzvapvgWHqjhSXc+49CS7Q9kHHfm7Ncb0XCKyQVVz2isXyG6ovcBQr9cZnmNtWQJ8qZPXdllslItGt1Lf2H2D3MYY01sEMizWAWNEJFNEonEGrJd7FxCRMV4vLwG2eZ4vBxaISIyIZAJjgA8DWFfiolwAQRu3MMaYnixgYxaq2iAitwKrABfwuKpuFpF7gPWquhy4VURmAfXAIeBaz7WbRWQpsAVoAG7pykwoX8RGOblZU99IUpzt02CMMd4CelOeqq4AVrQ49jOv5985wbX3AvcGrnbHckVEEO2K6NY7uY0xprew5T68BHNGlDHG9GQWFl5io1zUNTTidtsgtzHGeLOw8BIXFYECNQ3+a13MmDHjuJvsHnzwQW6++eY2r0lISABg3759zJs3r9Uy5513Hi2nCrf04IMPUlVV1fz64osv5vDhw75W3RhjmllYeIltnhHlv3GLhQsXsmTJkmOOLVmyhIULF7Z77eDBg1m2rM1bS9rVMixWrFhBv379Ov1+xpjwZWHhJRDLfsybN4+XX365ebOjnTt3sm/fPiZOnMjMmTOZNGkSp5xyCv/617+Ou3bnzp2cfPLJAFRXV7NgwQKys7O54oorqK6ubi538803Ny9vfvfddwPwhz/8gX379jFjxgxmzJgBwIgRIygpKQFg8eLFnHzyyZx88sk8+OCDzZ+XnZ3NTTfdxPjx47nwwguP+RxjTPgKmyXKWXkn7P/0hEUEGNUUFJ5WxgmlnQJz7jthkQEDBjBlyhRWrlzJ3LlzWbJkCfPnzycuLo4XXniBpKQkSkpKOOOMM7j88svbvHv8T3/6E3369CE3N5dNmzYxadKk5nP33nsvAwYMoLGxkZkzZ7Jp0yZuu+02Fi9ezOuvv05KSsox77VhwwaeeOIJPvjgA1SVqVOncu6559K/f3+2bdvGs88+y2OPPcb8+fN5/vnnueaaa9r/uzDGhDRrWbQQIc5S5Yr/Brm9u6KauqBUlR/96EdMmDCBWbNmsXfvXg4cONDme7z11lvNX9oTJkxgwoQJzeeWLl3KpEmTmDhxIps3b251eXNv77zzDldccQXx8fEkJCRw5ZVX8vbbbwOQmZnJaaedBtgy6MaYo8KnZdFOC6BJeUUt+w5Xk52WRFSkf7J07ty5fPe73+Wjjz6iqqqKyZMn87e//Y3i4mI2bNhAVFQUI0aMaHVZ8vZ88cUXPPDAA6xbt47+/ftz3XXXdep9msTExDQ/d7lc1g1ljAGsZXGcpkHuaj/OiEpISGDGjBl8/etfbx7YPnLkCAMHDiQqKorXX3+dXbt2nfA9pk+fzjPPPAPAZ599xqZNmwBnefP4+Hj69u3LgQMHWLlyZfM1iYmJlJeXH/de55xzDi+++CJVVVVUVlbywgsvcM455/jrxzXGhKDwaVn46JhlP2L9t+zHwoULueKKK5q7o66++mouu+wyTjnlFHJycsjKyjrh9TfffDPXX3892dnZZGdnM3nyZABOPfVUJk6cSFZWFkOHDj1mefNFixYxe/ZsBg8ezOuvv958fNKkSVx33XVMmeJsEXLjjTcyceJE63IyxrQpYEuUd7euLlHuLa+wjD7RkQxL7uOv6oUcW6LcmNDQE5Yo77Vio1x+vTHPGGN6OwuLVsRGuaitd9uyH8YY4xHyYdGZbrbYqAgUpdZaF60Kla5LY4zvQjosYmNjKS0t7fCXW/OMKFuu/DiqSmlpKbGxscGuijGmG4X0bKiMjAwKCgooLi7u0HWqUHykmsqiSPrZRkjHiY2NJSMjI9jVMMZ0o5AOi6ioKDIzMzt17V1/fIeE2EievvEMP9fKGGN6n5DuhuqKrLQkcgvLrX/eGGOwsGhTVnoiByvrKC6vDXZVjDEm6Cws2pCdngTAlsKyINfEGGOCz8KiDdlpTljk7T9+bSVjjAk3FhZt6NsnisF9Y8m1loUxxlhYnEhWehJ5hdayMMaYgIaFiMwWka0isl1E7mzl/O0iskVENonIGhEZ7nWuUUQ2eh7LA1nPtmSnJ7KjuMLu5DbGhL2AhYWIuIBHgDnAOGChiIxrUexjIEdVJwDLgPu9zlWr6mmex+WBqueJZKUl0eBWthdVBOPjjTGmxwhky2IKsF1V81W1DlgCzPUuoKqvq2qV5+VaoEfdFpydnghgXVHGmLAXyLAYAuzxel3gOdaWG4CVXq9jRWS9iKwVkS+1doGILPKUWd/RJT18MSI5npjICBvkNsaEvR6x3IeIXAPkAOd6HR6uqntFZCTwmoh8qqo7vK9T1UeBR8HZ/Mjf9Yp0RTB2UKJNnzXGhL1Atiz2AkO9Xmd4jh1DRGYBPwYuV9Xm26VVda/nz3zgDWBiAOvapuz0RHILy2zZD2NMWAtkWKwDxohIpohEAwuAY2Y1ichE4M84QVHkdby/iMR4nqcA04AtAaxrm7LSkiitrKO4wpb9MMaEr4B1Q6lqg4jcCqwCXMDjqrpZRO4B1qvqcuC3QALwnIgA7PbMfMoG/iwibpxAu09VgxIWTct+5BWWMzDR9nAwxoSngI5ZqOoKYEWLYz/zej6rjeveA04JZN18lZXmzIjKLSxj+tjUINfGGGOCw+7gbkf/+GjSkmJtkNsYE9YsLHzQNMhtjDHhysLCB1npSWwvqqCuwfbkNsaEJwsLH2Sn27IfxpjwZmHhg2zPIHfefuuKMsaEJwsLH2SmxBMdGWGD3MaYsGVh4QNn2Y8EG+Q2xoQtCwsfZaUlkWurzxpjwpSFhY+y05MoqailuNyW/TDGhB8LCx/ZILcxJpxZWPgoy7NGlI1bGGPCkYWFjwbERzMoKcZ2zTPGhCULiw7ITk8i16bPGmPCkIVFB2SlJbG9qNyW/TDGhB0Liw7ITk+kvlHJL7FlP4wx4cXCogOybZDbGBOmLCw6IDMlnmhXhA1yG2PCjoVFB0S5IhgzKMEGuY0xYcfCooOcZT+sG8oYE14sLDooOz2R4vJaSips2Q9jTPiwsOigpkFuG7cwxoQTC4sOyrI1oowxYcjCooOSE2IYmBjDFhu3MMaEEQuLTshKT7JuKGNMWAloWIjIbBHZKiLbReTOVs7fLiJbRGSTiKwRkeFe564VkW2ex7WBrGdHZacnsr2ogvpGW/bDGBMeAhYWIuICHgHmAOOAhSIyrkWxj4EcVZ0ALAPu91w7ALgbmApMAe4Wkf6BqmtHZaclUdfoJr+4MthVMcaYbhHIlsUUYLuq5qtqHbAEmOtdQFVfV9Uqz8u1QIbn+UXAalU9qKqHgNXA7ADWtUOaZ0TZILcxJkwEMiyGAHu8Xhd4jrXlBmBlR64VkUUisl5E1hcXF3exur4bmRpPlEtskNsYEzZ6xAC3iFwD5AC/7ch1qvqoquaoak5qampgKteKKFcEowcm2iC3MSZsBDIs9gJDvV5neI4dQ0RmAT8GLlfV2o5cG0zZ6Ym27IcxJmwEMizWAWNEJFNEooEFwHLvAiIyEfgzTlAUeZ1aBVwoIv09A9sXeo71GNlpSRSV11Jqy34YY8JAwMJCVRuAW3G+5HOBpaq6WUTuEZHLPcV+CyQAz4nIRhFZ7rn2IPBLnMBZB9zjOdZjNA1yb7UVaI0xYSAykG+uqiuAFS2O/czr+awTXPs48Hjgatc1WenOsh9bCss4a3RKkGtjjDGB1SMGuHujlIQYUhNjyLOWhTEmDFhYdEFWmg1yG2PCg4VFF2SnJ7HtQAUNtuyHMSbEWVh0QXZ6orPsR4kt+2GMCW0WFl2QlebMiLKuKGNMqLOw6IJRqQlEucQGuY0xIc/CoguiIyMYlZpgLQtjTMizsOiicbYRkjEmDFhYdFFWeiL7y2o4VFkX7KoYY0zAWFh0UfMgt+1tYYwJYRYWRwrg75fBznc6dXnTGlG51hVljAlhFhZ9kqEoD968v1OXpybGkJIQTZ4NchtjQpiFRVQcTLsNvngT9nzYqbfITk+y6bPGmJDmU1iIyCgRifE8P09EbhORfoGtWjfK+brTwuhk6yIrLZGtB8pt2Q9jTMjytWXxPNAoIqOBR3F2sXsmYLXqbtHxcOYtsH017P2ow5dnpydR1+BmZ6kt+2GMCU2+hoXbs5nRFcDDqnoHkB64agXB6TdBbD9464EOX9o0I2qLDXIbY0KUr2FRLyILgWuBlzzHogJTpSCJTYIzboatL8P+zzp06aiB8URGiA1yG2NClq9hcT1wJnCvqn4hIpnAPwJXrSCZ+g2IToS3ftuhy2IiXYweaMt+GGNCl09hoapbVPU2VX1WRPoDiar6mwDXrfvF9Yepi2DLv6B4a4cuzUpLtBlRxpiQ5etsqDdEJElEBgAfAY+JyOLAVi1IzrgFovp0eOwiOz2JwiM1HK6yZT+MMaHH126ovqpaBlwJPKmqU4FZgatWEMUnw+lfh8+WQekOny/Lsju5jTEhzNewiBSRdGA+Rwe4Q9eZ3wZXNLzte+MpOz0RgDxbI8oYE4J8DYt7gFXADlVdJyIjgW2Bq1aQJQ6CydfBpiVwaJdPl6QmxJAcH22D3MaYkOTrAPdzqjpBVW/2vM5X1S+3d52IzBaRrSKyXUTubOX8dBH5SEQaRGRei3ONIrLR81ju6w/kN2fdBhIB7z7oU3ERISvdBrmNMaHJ1wHuDBF5QUSKPI/nRSSjnWtcwCPAHGAcsFBExrUothu4jtbvBq9W1dM8j8t9qadf9R0Cp10NHz8FZft8uiQ7LYmt+23ZD2NM6PG1G+oJYDkw2PP4t+fYiUwBtntaIXXAEmCudwFV3amqm4Ce+e169ndB3fDuQz4Vz0pPorbBzc7SqgBXzBhjupevYZGqqk+oaoPn8TcgtZ1rhgB7vF4XeI75KlZE1ovIWhH5UmsFRGSRp8z64uLiDry1j/oPhwkLYMPfoPxAu8VtkNsYE6p8DYtSEblGRFyexzVAaSArBgxX1Rzgq8CDIjKqZQFVfVRVc1Q1JzW1vezqpHNuh8Y6eP/hdouOHphAZITYILcxJuT4GhZfx5k2ux8oBObhjDWcyF6c1WmbZHiO+URV93r+zAfeACb6eq1fJY+Ck+fBuseh8sT5GBPpYlRqAnl2r4UxJsT4Ohtql6perqqpqjpQVb8EtDcbah0wRkQyRSQaWIAz7tEuEenvtX9GCjAN2OLLtQEx/ftQXwVrH2m3aFZ6orUsjDEhpys75d1+opOeJc1vxbk/IxdYqqqbReQeEbkcQEROF5EC4CrgzyKy2XN5NrBeRD4BXgfuU9XghUXqSTBuLnzwKFQfOmHRrLQk9h2p4UhVfTdVzhhjAi+yC9dKewVUdQWwosWxn3k9X4fTPdXyuveAU7pQN/+bfgdseRE++DOcd9wtI828B7mnjkzurtoZY0xAdaVloX6rRW+QdjKcdAms/V+oabubKbt5jSjrijLGhI4ThoWIlItIWSuPcpz7LcLLuXdAzRFY91ibRQYmxjAgPtru5DbGhJQThoWqJqpqUiuPRFXtShdW7zR4Ioy+AN5/BOpa329bRMhKs0FuY0xo6Uo3VHiafgdUlcL6x9sskp2exNYD5TS6w6unzhgTuiwsOmrYVMicDu/+AeqrWy2SlZZITb2bnaWttz6MMaa3sbDojOk/gMoi+Kj1bcibBrnt5jxjTKiwsOiMEWfDsDOd5csbao87PXpgAq4IsTWijDEhw8KiM0ScsYuyvbDx+NXVY6NcjEyJt0FuY0zIsLDorFHnw5DJ8M5iaDz+bu3s9CTbj9sYEzIsLDpLxBm7OLwbNi097nRWeiJ7D1dzpNqW/TDG9H4WFl0x9iJImwBv/w7cjcecahrk3mo35xljQoCFRVc0jV0c3AGf/fOYU9lptuyHMSZ0WFh0VdalMHAcvP0AuI/uDjsoKYZ+faJsRpQxJiRYWHRVRASc8z0ozoPco9t1iAjZaTbIbYwJDRYW/jD+CkgeDW89AHp0iY+s9ES27rdlP4wxvZ+FhT9EuJzWxYFPYevK5sPZ6UlU1zey+2BVECtnjDFdZ2HhL6dcBf2Gw1v3N7cubJDbGBMqLCz8xRUF59wO+z6G7WsAGDMogQiBPAsLY0wvZ2HhT6d+FZIymlsXsVEuRqYmsMUGuY0xvZyFhT9FRsPZ/w17PoCdbwPOcuU2fdYY09tZWPjbxK9BQhq8eT/gDHIXHKqmrMaW/TDG9F4WFv4WFQvTbnNaFrvXkp2eCNiyH8aY3s3CIhAmXw99UuDN+702QrKuKGNM7xXQsBCR2SKyVUS2i8idrZyfLiIfiUiDiMxrce5aEdnmeVwbyHr6XXQfOOtW2LGGtPLN9I2LskFuY0yvFrCwEBEX8AgwBxgHLBSRcS2K7QauA55pce0A4G5gKjAFuFtE+geqrgFx+o0Q1x956wGy022Q2xjTuwWyZTEF2K6q+apaBywB5noXUNWdqroJcLe49iJgtaoeVNVDwGpgdgDr6n8xiXDGt+DzlZyXtJ+t+8tx27IfxpheKpBhMQTY4/W6wHMs0Nf2HFMWQUwSlxx+mqo6W/bDGNN79eoBbhFZJCLrRWR9cXFxsKtzvLh+MGURQ/evZowUWFeUMabXCmRY7AWGer3O8Bzz27Wq+qiq5qhqTmpqaqcrGlBnfAuNiufbkS/aILcxptcKZFisA8aISKaIRAMLgOXtXNNkFXChiPT3DGxf6DnW+8QnI6ffwCWutRzctTnYtTHGmE4JWFioagNwK86XfC6wVFU3i8g9InI5gIicLiIFwFXAn0Vks+fag8AvcQJnHXCP51jvdNa3aZQophX+Pdg1McaYThHV0Jihk5OTo+vXrw92Ndr0yWPfZHzB/1Fz8zoS0kYHuzrGGAOAiGxQ1Zz2yvXqAe7epHzSzTQSQfVrDwS7KsYY02EWFt0kc9RYljaex4Btz8GRgmBXxxhjOsTCopsM7hvLU5FXOLvovftQsKtjerrDu+HZhbD5xWDXxBgAIoNdgXAhIvRNH8Xrh2Yya91fYO9HMOwMGDrV+TNhYLCraHqKfRvhmflQccDZ073ifpi6KNi1MmHOwqIbjUtP4qd7v8LMc8Yju9+HDx+D9//onOyfeWx4pJwEEdbwCzufvwLPXQd9BsCiN+DN38LKO5zgOP8nIBLkCppwZWHRjbLSEvlbXRy7J93B8AvioaHW+S1yz1rY/QFsewU+edYpHNsPhk45Gh6DJzmr2ZrQtf5xePl7kHYKfHUpJKbB/Cfh5dvh7QecwLj0QXDZ/7am+9m/um7UtLdFbmE5w5PjITIGhk11HtNwxjNKd3jCY62zPeu2V5yLIyIh/VQYeoZTfugZkDgoeD+M8R+3G167B975PYy5EOY9ATEJzjlXJFz2ECQMcvZ2ryqFeY9DVFxw62zCjoVFNxo7KBERyC0sY/bJaccXEIGU0c5j4jXOsaqDTmg0hce6v8DaR5xz/UccGx6pWdZ11ds01MKLN8Nnz0PO12HOb49vOYjA+T92xrVW3AH/uAIWPgtxvWvVftO7WVh0o7hoF5nJ8R1bULDPADhpjvMA58ul8JOj4bH9Vdi0xDkX2xcyphwNjyGTreuqJ6s6CEuuht3vwaxfwLTvnHhMYspNEJ8C/1wEj8+Br/0TkgZ3X31NWLOw6GbZ6Ul8tu9I598gMsYzljHFea0KB/M94eEZ+9i+2jkXEQlpE2DU+TDhK5A6tus/gPGPg1/A01fB4V1Ot9LJX/btuvFXQNwAJ2T+eiFc80/772q6hS330c0eXrON363+nN/Om8CXJ2UQERGA2S1VB2HPh8eOfajbGSQ/dYHzxRSf4v/P7W5uN9SVOy2q3mTvBnjmK9BY73QnDT+r4+9R+Ak8NQ/c9XD1Mshod7UGY1rl63IfFhbdrKishm88tYGPdx/m1KH9uPuycUwaFuC+5/L98Okyp7tq/6dOi2P0LKe1cdKc3jVYWl8N+W9C3kvw+X+cAd+J18CMn/SOAf+8l2HZDc74w9XLutYqOJgP/7jSmSU1/0kYc4H/6mnChoVFD+Z2Ky9u3Mt9K/MoKq/lyolD+OGcLAYlxQb+ww9scUJj01IoL4SYJBg312lxDDurZw6QVx10giHvZdjxGtRXQXSi8+UY1x8+ehJc0XD2d+HMW3ruOM0Hj8LKH8DgifDV//PPjZgVRfD0PNj/Gcx9BE5b2PX3NJ1XX+PMWtvwN2hsANTpKgbnOXheez+na+UAhuTATWs6VWULi16gsraB/31jO4+99QWRLuGWGaO54exMYqNcgf9wdyN88RZs+j/YshzqK6HvMJhwFUxYEPx+8INfwNYVTkDsft/pRksc7LSEsi6BEedAZLRTtnQHvHo35P7bKTPzZ06rqacEn9sNq3/q3IB50iXw5ccgOt5/719TBv93DXzxJlzwS5h2m//e2/huz4fwr1ug5HPIvgz6Nu3f5ulq9p680Pzc+1zLcr6ew5noMPm6TlXbwqIX2V1axa9e3sIrWw4wbEAffnxJNheOG4R01926dZXOl/InSyD/9eCMb7jdUPgx5K1wQqKSNtSMAAAXgklEQVRoi3N84HjIuhhOutj5jfxEfye73oNVP4Z9Hzn3pFx4L2SeE/i6n0h9tTN7KXc5TPkGzP41RATgl4GGWnjhG7D5BTjzVic0ekpYhrq6SljzS/jg/zkBcdmDMHpmsGvlMwuLXuidbSXc89JmPj9QwbTRyfzs0vGclJbYvZXozvGNhlrY+bYTVFtXOt1iEuF0h2Vd4nzegMyOvafbDZ8tg1d/AWUFTshccA+kjPFfvX1VWeIsBliwDi66F874VmCX63C74T93wod/hlPmO91STa0vExj5b8Dy25xZbaffBLPuhphu/n+2iywseqmGRjdPf7Cbxas/p6K2gWumDuO7F4ylX58g/E8fiPGN6sOwbTVsfRm2verMZoqKh9HnO100Yy9y7i3pqvpqWPu/8PbvoaEacm6Ac38I8cldf29flO5wxhLK9sGVjzp/b91BFd5ZDGvugVEznYHvprvBjf/UHIFXfuKMlw0YBXP/2LlZbT2AhUUvd6iyjsWrP+fpD3aRFBfF9y4Yy8Ipw4h0BaFroavjG4f3HB1/2PUuuBsgfiCcNBuyLoXMcyEqQIP7FcXwxv84A47RiTD9+zD1G879KoGy50Nnaiw4A9lN98R0p4+ehH9/B9JPg6ufC42p0j3F1pXw0nedWWhnfRvOu6t3zShswcIiROTtL+MXy7fwfn4pWWmJ/OyycZw1Koj/4/syvqHqdGE1BcT+Tc61KWOdbqGsS5zZG93Zp16UC6/81Llhsd9wmPVz5wY3f3cLbfmXM0aRNNiZGps8yr/v3xF5K2DZ9dA3w7l5r//w4NUlFFSWwn9+CJ8+54ylzf0jDJkU7Fp1mYVFCFFVVm3ez69ezqXgUDWzx6fx40uyGTogyFNEWxvfGHE2lObDkd2AOL9VZ13idDGl9IC9x3e8Bqt+AkWbnaVRLrrXP7/5qzrdXqt+DBmnw8Il3dfldSK71zp7Y0TGwTXPQ9rJwa5R76MKm/8JK37gdD9Nv8OZph0i40EWFiGopr6Rv7ydzyOv76BRlUXnjORbM0bRJ7oHrNrSNL6RtwKSRzszmMbO7pmbOrkbYePT8NqvnK6E8Vc4LY3+Izr/fv+5yxlYzr7cGaPoSd0SRbnOzXt1lc4d4yOmBbtGvUdZobNs/NaXnRb03Edg0Lhg18qvLCxC2P4jNdy3MpcXN+4jLSmWO+dkMfe0wd031TZU1FY4W9y+9zBoI0z9JpzzPYjr5/t71FXB8zc6XyY9ecrq4T3w1JVwyLMWVfalwa5Rz6YKHz/ltBQba2HGj53ZbCG4l4iFRRjYsOsgP1++hU/3HmHSsH78/PLxTMjowBedcRzZ67QyPvEs+33eXZBzPbiiTnxdRZEzkL3vY5jzG2fgvCerOuh0Se3dAJcsdn5Gc7xDu5zJAfmvw/BpcPnDwR17CjALizDhdivLNhRw/6o8SivrmDcpgztmn8TAxG5YOiTU7NvoTIfc+TYkj3HuzzhpTuuD4CXb4KkvO4Ex76/OuExvUFfpbNu67RU470dw7g9sq9Ymbjese8y5R0cELvgFTP56z2wp+lGPCAsRmQ08BLiAv6jqfS3OxwBPApOBUuArqrpTREYAucBWT9G1qvrNE31WuIZFk/Kaeh5+bTtPvPsFMZEuvn3+aK6flkl0ZGj/Q/c7VWdq5OqfQul2Z1mRi+517ghvsus952Y7VxQs/D/ImBy8+nZGY71zI9knz8DpN8Kc+wNzV3lvUrINln/bWVpm9Cxn+9p+Q9u/LgQEPSxExAV8DlwAFADrgIWqusWrzLeACar6TRFZAFyhql/xhMVLqurz1I1wD4sm+cUV3PtyLmvyishMiecnl2RzftZAG8/oqMZ6WP8EvPFrqD5Ezfj5/Nn1VQaXbWTennuRfsOd+xc6eod5T6HqrKf17kPODYNXPBq4e116ssYGeP9heP3XzqSE2b+GUxeGVWurJ4TFmcDPVfUiz+u7AFT1115lVnnKvC8ikcB+IBUYjoVFl7yxtYh7XtpCfnElp4/oz4LThzHnlLSeMXOqF6mtOEje0p+RvftZ3CrESj3r3FmsOmUxN1w4ifS+PWjWU2e8/wis+pHTglrwdO/bG6Qr9n/mLPxXuNFZ+O/i3/WOZe79rCeExTxgtqre6Hn9NWCqqt7qVeYzT5kCz+sdwFQgAdiM0zIpA36iqm+38hmLgEUAw4YNm7xr166A/Cy9VX2jm6fW7uLv7+1kZ2kVCTGRXDohnatyhjJpWD9rbZxA070tv16Zx67SKq4a2cDP4l/AFRPP/RHX8/SGIgThq1OH8a3zRjGwO5aXD5RNS519wJOGONOeIyKdhyvy6PPWHsedd0FE1LGvXVFtn4+Oh/hUSEiF2H7d99t8Qy289YCzLEpcf7j4ARj/pe757B6ot4dFOZCgqqUiMhl4ERivqm1uXm0ti7apKut2HmLp+j2s+LSQqrpGRqXGc1XOUK6cOKR3f9EFwGd7j/DLl7bwwRcHGTMwgZ9cOo5zx6YeU6bgUBV/fG07z20oIDJC+NoZw/nmeaNISQjgMiKBtOM1eHuxs6aWu965d8TdcPTR2HDsa3ejp5zndVdFRDl3/8enOEvBxKc6zxOanqd6nUvp/HItBRuc1kRxrrM45uz7/LMWWS/WE8Ki091Q2qJSIvIG8H1VbTMNLCx8U1HbwIpNhSxdv4f1uw7hihBmnJTKVTlDOT9rIFHBWHuqhygqq+G3q7ay7KMC+veJ5rsXjGXh6UNPuB7XrtJKHn5tO//8qICYSBf/ddZwvjF9FAPiQ+PuXp+oOsu+uBucsZ7mMGkKljbCp67cWZm3oggqi49/VBQ7i0C2JqavV5ikeAWK16PpXGw/JwRfv9e5yz4x3RnAHnth9/499VA9ISwicbqRZgJ7cQa4v6qqm73K3AKc4jXAfaWqzheRVOCgqjaKyEjgbU+5g219noVFx+0ormDZhgKe31BAUXktyfHRXDFxCFflDO3+pdGDqKa+kcfeyudPb+6gvtHN9dMyuWXGaPrGtXOfhZf84gr+sGYb//pkH32iXFw/LZObzhlJ3z6+v4dpQdWZ6ttaiLQWLlUHOWb3uCYRUU5LpK4CJl/vTImOTer2H6enCnpYeCpxMfAgztTZx1X1XhG5B1ivqstFJBb4BzAROAgsUNV8EfkycA9QD7iBu1X13yf6LAuLzmtodPPWtmKeW1/Aq7kHqG9UTs3oy1U5Q7ns1MEd+tLsTVSV5Z/s4zcr89h3pIbZ49O46+Ishid3fhe7bQfKeXDNNl7eVEhiTCQ3nJPJ18/OJCk2NP8Oe5TGBmdP9spiqCxyWi1NQVJ9GE6+EjKnB7uWPU6PCIvuZGHhH6UVtby4cR/Prd9D3v5yYiIjmH1yGvNzhnLmyGQiIkJjUHzDrkP88qUtbNxzmPGDk/jppeM4Y6T/Fv7LLSzjwVc/Z9XmA/SNi2LR9JFce9YIEmJsNprpWSwsTJeoKp/tLWPp+j38a+NeymoaGNIvjqtyMvjypIzgr3jbSQWHqvjNf7by70/2MTAxhjsuOokvT8oIWAh+tvcIv1/9OWvyihgQH803po/ka2cOtynMpsewsDB+U1PfyKrN+1m2oYB3tpegCtNGJzM/ZygXjU8jNqrn3/1bWdvAn97YwWNv5wPwjekj+ca5o4jvpt/0N+45zO9Xf86bnxeTkhDNN88dxTVnDO8Vf3cmtFlYmIAoOFTF8xv2suyjPew5WE1ibCSXnzqY+TlDmZDRt8fdu9HoVp7fUMBvX9lKcXktc08bzA9mZzGkX3Buplu/8yC/f/Vz3t1eysDEGG6ZMZoFU4YSE2mhYYLDwsIElNutrP2ilOfWF7Di00JqG9yMHZTAVZOHMiVzACNT40kM8qDu+ztK+eVLW9hSWMakYf346aXjmDisf1Dr1GRtfimLX/mcD3ceJL1vLLeeP5qrJg+1tbxMt7OwMN2mrKaelz5x7t3YuOdw8/HUxBhGpcYzMjWBkSnxjEpNYFRqAkP6x+EK4ED5zpJK/mdFLq9sOcCQfnH8cE4Wl01I73GtHlXl3e2l/G71Vj7efZiM/nHcdv4Yrpg0JKzvdzHdy8LCBMWu0kpyC8vJL6kgv7iS/OIKdhRXcqS6vrlMdGQEI5L7MDIlgVED4xmZksBIT6h0ZZrukep6Hl6zjb+/v5NoVwTfmjGaG87O7PHjAqrKm58Xs3j152wqOMLw5D58Z+YY5p42JKChagxYWJgeRFU5WFlHfkklO4oqyC9xQiS/uJJdB6todB/9N5iSEMPI1HhGpTotkZGpTphk9I9r807qhkY3z364m8WrP+dwdT3zJw/lexeN7XV7eqgqa3KLWLz6c7YUljEyNZ75OUOZlT2QUakJPa5lZEKDhYXpFeoa3Ow+WOWER8nRlkh+cQWHqo62RqJcwojk+OYWyMiUeEYNTOBQZR33rcxjW1EFZ4wcwE8vHcf4wb175VS3W3lly37+940dbCo4AsDw5D6cnzWQWdmDOH3EABvbMH5jYWF6vUOVdeSXVLCjqJIdnm6tHcUV7C6tosGrNTIiuQ93XZzNheMGhdxv3/sOV7Mmr4jXcg/w7o5S6hrcJMZEMn1sKjOzBzLjpIH0D6d1qIzfWViYkFXf6GbPwSryiyuprm/kovFpYfGbdlVdA+9sK+G1vCLW5BVRXF5LhMCkYf2ZmT2IWdkDGT3QuqtMx1hYGBPC3G7l071HWJNXxJrcA2ze56zeP3RAHDOzBjErexBTMq27yrTPwsKYMFJ4pNppceQW8e72Emob3CTERDJ9bArnZw1ixkmpJPfWvTZMQFlYGBOmqusaeXd7CWvyDrAmt4ii8lrE013VNEg+dpB1VxmHhYUxBrdb2byvjFdzD/BaXhGf7nVmV2X0j2Nm1kBmZg9i6sgBttxIGLOwMMYcZ/+RGl7LK+K1vAO8s72Emno38dEuzhmTyoysVE4e0pfRAxMsPMKIhYUx5oSq6xp5b0eJZ2puEfvLagCIjBBGpSaQnZ5IdnpS8yM10cY8QpGFhTHGZ6rKjuJKcgvLvB7lzQECzt312emJjPMKkJGp8baOVS/na1jYDizGGESE0QMTGD0wgctOHdx8/FBlHbmFZWzxhEfe/jKeeHcndY1uAKJdEYwZlHC0BZLmtEbsRsHQYy0LY0yH1De6yfdqhTQFSUlFbXOZtKTY47qxMlPibWHEHshaFsaYgIhyRXBSWiInpSXypYlDmo8Xl9ce14319raS5qVZYqMiOGlQ4jHhkZIQQ0pCNAPio9tcKNL0DBYWxhi/SE2MITUxleljU5uP1TY0sr2ogtzC8uYQWbV5P0vW7TnmWhHo3yealIRoT4B4HonO61Sv18nxMUG/M93tVirqGiivaaC8pv6YP8tqGqitb6RPdCQJsZEkxLhIiIkiPsZFoufPhNjIXjfjzMLCGBMwMZEuxg/ue8xKwKrKgbJaCg5VUVJRS3FFHSXltZRUND3q+KTgMCXltVTWNbb6vkmxkaQkxngFiSdkEmOaWyspCTGkJsYct5+J262U13p/yR/7hV/WyrGW5SrqGuhqD360K6I5OOKjI0mMjSQ+JpIEr0d8jHO86XlC7LHnm453R3haWBhjupWIkNY3lrS+7e83Ul3X6AmUWk+g1HmFSi0l5c4AfHFFLeU1Da2+R0JMJAPio6lvdDtf9LWtl/MW5RISY6NIjHW+rBNjohie3Kf5WFJspNd5r3KxUSTFRhIT5aK6rpGK2noqahup8HxuRW0DFTX1VNY1Ul7TQGWt9/EGSivq2F1aRXmtc66qjbBsaeKwfrzwrWk+le2sgIaFiMwGHgJcwF9U9b4W52OAJ4HJQCnwFVXd6Tl3F3AD0AjcpqqrAllXY0zPExftYuiAPgwd0KfdsjX1jRysrDsmSIo9zw9W1hHlijjmCz2xjS/8pNgoYiIjurwcirPrY9c24Gp0K5V1TpBU1jZQXnvs80rP6wEJgZ99FrCwEBEX8AhwAVAArBOR5aq6xavYDcAhVR0tIguA3wBfEZFxwAJgPDAYeFVExqqqbzFrjAk7sVEuBveLY3C/uGBXxW9cEUJSbBRJsZ3fbthfAtnRNQXYrqr5qloHLAHmtigzF/i75/kyYKY4cT4XWKKqtar6BbDd837GGGOCIJBhMQTwnvJQ4DnWahlVbQCOAMk+XouILBKR9SKyvri42I9VN8YY461XT2xW1UdVNUdVc1JTU9u/wBhjTKcEMiz2AkO9Xmd4jrVaRkQigb44A92+XGuMMaabBDIs1gFjRCRTRKJxBqyXtyizHLjW83we8Jo6648sBxaISIyIZAJjgA8DWFdjjDEnELDZUKraICK3Aqtwps4+rqqbReQeYL2qLgf+CvxDRLYDB3ECBU+5pcAWoAG4xWZCGWNM8NhCgsYYE8Z8XUiwVw9wG2OM6R4h07IQkWJgVxfeIgUo8VN1ehr72XqvUP757GfrGYararvTSUMmLLpKRNb70hTrjexn671C+eezn613sW4oY4wx7bKwMMYY0y4Li6MeDXYFAsh+tt4rlH8++9l6ERuzMMYY0y5rWRhjjGmXhYUxxph2hX1YiMhsEdkqIttF5M5g18efRGSoiLwuIltEZLOIfCfYdfI3EXGJyMci8lKw6+JPItJPRJaJSJ6I5IrImcGukz+JyHc9/yY/E5FnRaRrW8oFkYg8LiJFIvKZ17EBIrJaRLZ5/uwfzDr6Q1iHhddufnOAccBCzy59oaIB+J6qjgPOAG4JsZ8P4DtAbrArEQAPAf9R1SzgVELoZxSRIcBtQI6qnoyzdtyC4NaqS/4GzG5x7E5gjaqOAdZ4XvdqYR0W+LabX6+lqoWq+pHneTnOF85xm0j1ViKSAVwC/CXYdfEnEekLTMdZaBNVrVPVw8Gtld9FAnGerQn6APuCXJ9OU9W3cBZC9ea9C+jfgS91a6UCINzDwqcd+UKBiIwAJgIfBLcmfvUg8APAHeyK+FkmUAw84eli+4uIxAe7Uv6iqnuBB4DdQCFwRFVfCW6t/G6QqhZ6nu8HBgWzMv4Q7mERFkQkAXge+G9VLQt2ffxBRC4FilR1Q7DrEgCRwCTgT6o6EagkBLoxmnj67+fihOJgIF5ErglurQLHs0dPr79HIdzDIuR35BORKJygeFpV/xns+vjRNOByEdmJ0314vog8Fdwq+U0BUKCqTa3AZTjhESpmAV+oarGq1gP/BM4Kcp387YCIpAN4/iwKcn26LNzDwpfd/HotERGcfu9cVV0c7Pr4k6repaoZqjoC57/ba6oaEr+dqup+YI+InOQ5NBNnI7BQsRs4Q0T6eP6NziSEBvA9vHcBvRb4VxDr4hcB2ymvN2hrN78gV8ufpgFfAz4VkY2eYz9S1RVBrJPxzbeBpz2/xOQD1we5Pn6jqh+IyDLgI5wZex/Ti5fHEJFngfOAFBEpAO4G7gOWisgNOFsnzA9eDf3DlvswxhjTrnDvhjLGGOMDCwtjjDHtsrAwxhjTLgsLY4wx7bKwMMYY0y4LC2M6QEQaRWSj18Nvd1aLyAjvlUuN6UnC+j4LYzqhWlVPC3YljOlu1rIwxg9EZKeI3C8in4rIhyIy2nN8hIi8JiKbRGSNiAzzHB8kIi+IyCeeR9NyFy4Recyz18MrIhIXtB/KGC8WFsZ0TFyLbqiveJ07oqqnAH/EWREX4GHg76o6AXga+IPn+B+AN1X1VJx1n5pWDhgDPKKq44HDwJcD/PMY4xO7g9uYDhCRClVNaOX4TuB8Vc33LN64X1WTRaQESFfVes/xQlVNEZFiIENVa73eYwSw2rNhDiLyQyBKVX8V+J/MmBOzloUx/qNtPO+IWq/njdi4oukhLCyM8Z+veP35vuf5exzdMvRq4G3P8zXAzdC8j3jf7qqkMZ1hv7UY0zFxXiv4grNPdtP02f4isgmndbDQc+zbODve3YGz+13T6rHfAR71rEraiBMchRjTQ9mYhTF+4BmzyFHVkmDXxZhAsG4oY4wx7bKWhTHGmHZZy8IYY0y7LCyMMca0y8LCGGNMuywsjDHGtMvCwhhjTLv+PxKRmc0d2BrBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluation = model.evaluate(x = X_test, y = y_test, verbose=1) \n",
    "loss = evaluation[0]\n",
    "acc = evaluation[1]\n",
    "print('LOSS ON TEST DATA='+str(loss))\n",
    "print('ACCURACY ON TEST DATA='+str(acc))\n",
    "\n",
    "\n",
    "# Plot training & validation loss values\n",
    "plt.plot(history.history['loss'], label='Train')\n",
    "plt.plot(history.history['val_loss'], label='Validation')\n",
    "plt.title('Model loss')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Validation'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00       484\n",
      "           1       0.99      0.97      0.98        95\n",
      "\n",
      "   micro avg       0.99      0.99      0.99       579\n",
      "   macro avg       0.99      0.98      0.99       579\n",
      "weighted avg       0.99      0.99      0.99       579\n",
      "\n",
      "Accuracy: 0.9930915371329879\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAF3JJREFUeJzt3X20XXV95/H3x4QHFQzVgNUkEBSoUlHBFHBYS5kRKTAaXNYHMqLFoaS10upobZ1VRyktWnW0YwccTYWiqCA41pXWWHxCMlrRxOGhgEIj8pCAw3MQwfDgd/7Y+5LjJdn35Hr3veeG92utuzh7n9/Z+7t/3JzP3ft3zm+nqpAkaWseN9MFSJJGm0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBoVklyVZLDJ2izZ5J7k8yZprJ6l+T6JEe0j09J8umZrkmPHQaFpkT7RnZ/+wb9/5KcnWSXqd5PVf1mVX1zgjY3VtUuVfXwVO+/fZN+sD3Ou5P8S5IXTvV+pFFiUGgqvbyqdgEOApYA7xrfII3Z/nv3ufY45wMXARfMcD1TLsncma5Bo2O2/4PVCKqqDcCXgecAJPlmktOSfBu4D3hGknlJzkxyS5INSf5q8FJRkpOS/CDJT5NcneSgdv3gJZiDk6xNck97FvPhdv3iJDX2Zpfk6UlWJrkzybokJw3s55Qk5yf5VLuvq5IsGfI4HwI+AyxIsvvANl+W5LKBM47nDjy3KMkXktyW5I4kp7frn5nkG+2625N8Jsluk+n/JMe2+78nyY+SHDW+7waO/dPj+uzEJDcC30jy5SQnj9v25Ule2T5+VpKvtv16TZLXTKZejT6DQlMuySLgGODSgdWvB5YDuwI3AGcDDwH7AAcCRwK/177+1cApwBuAJwFLgTu2sKuPAB+pqicBzwTO30pJ5wHrgacDrwLem+Q/DDy/tG2zG7ASOH3I49yxrfEO4K523YHAWcDvA08BPg6sTLJTG4T/1B7/YmBBu1+AAO9ra3w2sKjtg22S5GDgU8A72uN5EXD9Nmzixe3+fxs4F1g2sO39gb2ALyV5IvBV4LPAHsBxwEfbNtrOGBSaSl9McjfwLeBi4L0Dz51dVVe1f4U/mSZI3lpVP6uqW4G/oXmzgSYwPlBVa6qxrqpu2ML+HgT2STK/qu6tqkvGN2hD6zDgz6rq51V1GfAJmjf4Md+qqlXtmMY5wPMmOM7XtMd5P3AS8Kr2uKAJw49X1Xer6uGq+iSwCTgUOJgmCN7RHvfPq+pbAO0xfrWqNlXVbcCHad60t9WJwFnttn5RVRuq6ofb8PpT2truB/4BeH6SvdrnXgd8oao2AS8Drq+qv6+qh6rqUuB/A6+eRM0acQaFptIrqmq3qtqrqv6wfbMZc9PA472AHYBb2sszd9P85b1H+/wi4EdD7O9EYD/gh0nWJHnZFto8Hbizqn46sO4Gmr/mx/xk4PF9wM5J5iZ5XTtofW+SLw+0Ob+qdgOeClwJvGDcsb197LjaY1vU1rEIuGEgVB6R5KlJzmsvw90DfJpmDGRbDdt3W/PI/6e2z77E5gBfRnOpDZrjPGTccb4O+PVfYd8aUQ5YaboMTlN8E81f2fO39KbZPv/MCTdY9W/AsnZw/JXA55M8ZVyzm4EnJ9l1ICz2BDYMsf3PsPmNcUvP355kObA2yWer6pa29tOq6rTx7dtPR+2ZZO4Wjvu9NH10QFXdmeQVDHkJbJyuvvsZ8ISB5S29qY+fTvpc4D1JVgM70wzej+3n4qp66SRq1CzjGYWmXfuG+hXgQ0melORx7WDu2KWWTwB/kuQFzYekss/A5Y9HJDk+ye5V9Qvg7nb1L8bt6ybgX4D3Jdm5HVg+keYv9qk4lmuAC4E/bVf9HfAHSQ5pa39ikv+YZFfge8AtwF+363dOclj7ul2Be4GNSRbQjDFMxpnAG5O8pO3XBUme1T53GXBckh3aAftXDbG9VTRnD6fSfNprrH//Cdgvyevb7e2Q5LeSPHuSdWuEGRSaKW8AdgSuphkI/jzwNICqugA4jWag9KfAF2nGNcY7Crgqyb00A9vHjbvcNWYZzeDxzTTX3d9TVV+bwmP5ILA8yR5VtZZm3OL09rjWAScAtGMgL6cZwL+RZoD9te02/oLmY8UbaS73fGEyhVTV94A30oz5bKQZKxoL2f9Gc7ZxV7u/zw6xvU1tLUcMtm/Pzo6kuSx1M83lu/cDO02mbo22eOMiSVIXzygkSZ0MCklSJ4NCktTJoJAkdZp136OYP39+LV68eKbLkKRZ5fvf//7tVbX7xC0fbdYFxeLFi1m7du1MlyFJs0qSLU2DMxQvPUmSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTr0FRZKzktya5MqtPJ8kf5vmHsZXpL0nsiRptPR5RnE2zTTQW3M0sG/7sxz4Xz3WIkmapN6CoqpWA3d2NDkW+FR7T+RLgN2SPG2i7T7wwFRVKEkaxkyOUSzgl++jvJ5fvo/xI5IsT7I2ydpbbrlrWoqTJDVmxRQeVbUCWAHwjGcsqWuvneGCJGkEPPnJMH9+//uZyaDYACwaWF7IEDe8B1i9upd6JGnW2LSpCYply/rf10wGxUrg5CTnAYcAG6vqloleNHcuHHhg77VJ0ki74Qa4s2sUeAr1FhRJzgUOB+YnWQ+8B9gBoKo+BqwCjqG5+fx9NDeElySNmN6Coqo6T4iqqoA397V/SdLU8JvZkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI69RoUSY5Kck2SdUneuYXn90xyUZJLk1yR5Jg+65EkbbvegiLJHOAM4Ghgf2BZkv3HNXsXcH5VHQgcB3y0r3okSZPT5xnFwcC6qrquqh4AzgOOHdemgCe1j+cBN/dYjyRpEub2uO0FwE0Dy+uBQ8a1OQX4SpI/Ap4IHLGlDSVZDiwH2GOPPae8UEnS1s30YPYy4OyqWggcA5yT5FE1VdWKqlpSVUvmzdt92ouUpMeyPoNiA7BoYHlhu27QicD5AFX1HWBnYH6PNUmStlGfQbEG2DfJ3kl2pBmsXjmuzY3ASwCSPJsmKG7rsSZJ0jbqLSiq6iHgZOBC4Ac0n266KsmpSZa2zd4OnJTkcuBc4ISqqr5qkiRtuz4Hs6mqVcCqcevePfD4auCwPmuQJP1qZnowW5I04gwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktRp7rANkywA9hp8TVWt7qMoSdLoGCookrwfeC1wNfBwu7qAzqBIchTwEWAO8Imq+usttHkNcEq7vcur6j8NW7wkqX/DnlG8AviNqto07IaTzAHOAF4KrAfWJFlZVVcPtNkX+K/AYVV1V5I9hi9dkjQdhh2juA7YYRu3fTCwrqquq6oHgPOAY8e1OQk4o6ruAqiqW7dxH5Kkng17RnEfcFmSrwOPnFVU1R93vGYBcNPA8nrgkHFt9gNI8m2ay1OnVNU/D1mTJGkaDBsUK9ufPva/L3A4sBBYneSAqrp7sFGS5cBygD322LOHMiRJWzNUUFTVJ5PsSHsGAFxTVQ9O8LINwKKB5YXtukHrge+22/pxkmtpgmPNuP2vAFYA7LffkhqmZknS1BhqjCLJ4cC/0QxOfxS4NsmLJnjZGmDfJHu3IXMcjz4r+SLN2QRJ5tME0XXDFi9J6t+wl54+BBxZVdcAJNkPOBd4wdZeUFUPJTkZuJBm/OGsqroqyanA2qpa2T53ZJKxj92+o6rumPzhSJKm2rBBscNYSABU1bVJJvwUVFWtAlaNW/fugccFvK39kSSNoGGDYm2STwCfbpdfB6ztpyRJ0igZNijeBLwZGPs47P+hGauQJG3nhv3U0ybgw+2PJOkxpDMokpxfVa9J8q80czH9kqp6bm+VSZJGwkRnFG9p//uyvguRJI2mzu9RVNUt7cPbgZuq6gZgJ+B5wM091yZJGgHDTgq4Gti5vSfFV4DXA2f3VZQkaXQMGxSpqvuAVwIfrapXA7/ZX1mSpFExdFAkeSHN9ye+1K6b009JkqRRMmxQvJXmBkP/0E7D8Qzgov7KkiSNimG/R3ExcPHA8nVs/vKdJGk7NtH3KP5HVb01yT+y5e9RLO2tMknSSJjojOKc9r//ve9CJEmjqTMoqur77cO1wP1V9QuAJHNovk8hSdrODTuY/XXgCQPLjwe+NvXlSJJGzbBBsXNV3Tu20D5+Qkd7SdJ2Ytig+FmSg8YWkrwAuL+fkiRJo2TY+1G8Fbggyc1AgF8HXttbVZKkkTHs9yjWJHkW8Bvtqmuq6sH+ypIkjYqhLj0leQLwZ8BbqupKYHESpx6XpMeAYcco/h54AHhhu7wB+KteKpIkjZRhg+KZVfUB4EGAdibZ9FaVJGlkDBsUDyR5PO00HkmeCWzqrSpJ0sgY9lNP7wH+GViU5DPAYcAJfRUlSRodEwZFkgA/pLlp0aE0l5zeUlW391ybJGkETBgUVVVJVlXVAWy+aZEk6TFi2DGK/5vkt3qtRJI0koYdozgEOD7J9cDPaC4/VVU9t6/CJEmjYdig+O1eq5AkjayJ7nC3M/AHwD7AvwJnVtVD01GYJGk0TDRG8UlgCU1IHA18qPeKJEkjZaJLT/u3n3YiyZnA9/ovSZI0SiY6o3hkhlgvOUnSY9NEQfG8JPe0Pz8Fnjv2OMk9E208yVFJrkmyLsk7O9r9TpJKsmRbD0CS1K/OS09VNWeyG04yBzgDeCmwHliTZGVVXT2u3a7AW4DvTnZfkqT+DPuFu8k4GFhXVddV1QPAecCxW2j3l8D7gZ/3WIskaZL6DIoFwE0Dy+vbdY9o78O9qKo6pwZJsjzJ2iRrN268beorlSRtVZ9B0SnJ44APA2+fqG1VraiqJVW1ZN683fsvTpL0iD6DYgOwaGB5YbtuzK7Ac4BvtlODHAqsdEBbkkZLn0GxBtg3yd5JdgSOA1aOPVlVG6tqflUtrqrFwCXA0qpa22NNkqRt1FtQtN+7OBm4EPgBcH5VXZXk1CRL+9qvJGlqDTsp4KRU1Spg1bh1795K28P7rEWSNDkzNpgtSZodDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR16jUokhyV5Jok65K8cwvPvy3J1UmuSPL1JHv1WY8kadv1FhRJ5gBnAEcD+wPLkuw/rtmlwJKqei7weeADfdUjSZqcPs8oDgbWVdV1VfUAcB5w7GCDqrqoqu5rFy8BFvZYjyRpEvoMigXATQPL69t1W3Mi8OUtPZFkeZK1SdZu3HjbFJYoSZrISAxmJzkeWAJ8cEvPV9WKqlpSVUvmzdt9eouTpMe4uT1uewOwaGB5YbvulyQ5Avhz4MVVtanHeiRJk9DnGcUaYN8keyfZETgOWDnYIMmBwMeBpVV1a4+1SJImqbegqKqHgJOBC4EfAOdX1VVJTk2ytG32QWAX4IIklyVZuZXNSZJmSJ+XnqiqVcCqcevePfD4iD73L0n61Y3EYLYkaXT1ekYhSerPpk1w7bX978egkKRZaN48+MlPYPXqYV+x6xMnuy+DQpJmod12g0MO2ZZXzJkz2X05RiFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSerUa1AkOSrJNUnWJXnnFp7fKcnn2ue/m2Rxn/VIkrZdb0GRZA5wBnA0sD+wLMn+45qdCNxVVfsAfwO8v696JEmT0+cZxcHAuqq6rqoeAM4Djh3X5ljgk+3jzwMvSZIea5IkbaO5PW57AXDTwPJ64JCttamqh5JsBJ4C3D7YKMlyYHm79OCSJb92fS8Vzzqb5sFOG2e6itFgX2xmX2xmX2x2z8LJvrLPoJgyVbUCWAGQZG3VXUtmuKSR0PTFffYF9sUg+2Iz+2KzJGsn+9o+Lz1tABYNLC9s122xTZK5wDzgjh5rkiRtoz6DYg2wb5K9k+wIHAesHNdmJfC77eNXAd+oquqxJknSNurt0lM75nAycCEwBzirqq5KciqwtqpWAmcC5yRZB9xJEyYTWdFXzbOQfbGZfbGZfbGZfbHZpPsi/gEvSeriN7MlSZ0MCklSp5ENCqf/2GyIvnhbkquTXJHk60n2mok6p8NEfTHQ7neSVJLt9qORw/RFkte0vxtXJfnsdNc4XYb4N7JnkouSXNr+OzlmJursW5Kzktya5MqtPJ8kf9v20xVJDhpqw1U1cj80g98/Ap4B7AhcDuw/rs0fAh9rHx8HfG6m657Bvvj3wBPax296LPdF225XYDVwCbBkpuuewd+LfYFLgV9rl/eY6bpnsC9WAG9qH+8PXD/TdffUFy8CDgKu3MrzxwBfBgIcCnx3mO2O6hmF039sNmFfVNVFVXVfu3gJzXdWtkfD/F4A/CXNvGE/n87iptkwfXEScEZV3QVQVbdOc43TZZi+KOBJ7eN5wM3TWN+0qarVNJ8g3ZpjgU9V4xJgtyRPm2i7oxoUW5r+Y8HW2lTVQ8DY9B/bm2H6YtCJNH8xbI8m7Iv2VHpRVX1pOgubAcP8XuwH7Jfk20kuSXLUtFU3vYbpi1OA45OsB1YBfzQ9pY2cbX0/AWbJFB4aTpLjgSXAi2e6lpmQ5HHAh4ETZriUUTGX5vLT4TRnmauTHFBVd89oVTNjGXB2VX0oyQtpvr/1nKr6xUwXNhuM6hmF039sNkxfkOQI4M+BpVW1aZpqm24T9cWuwHOAbya5nuYa7MrtdEB7mN+L9cDKqnqwqn4MXEsTHNubYfriROB8gKr6DrAzMH9aqhstQ72fjDeqQeH0H5tN2BdJDgQ+ThMS2+t1aJigL6pqY1XNr6rFVbWYZrxmaVVNejK0ETbMv5Ev0pxNkGQ+zaWo66azyGkyTF/cCLwEIMmzaYLitmmtcjSsBN7QfvrpUGBjVd0y0YtG8tJT9Tf9x6wzZF98ENgFuKAdz7+xqpbOWNE9GbIvHhOG7IsLgSOTXA08DLyjqra7s+4h++LtwN8l+S80A9snbI9/WCY5l+aPg/nteMx7gB0AqupjNOMzxwDrgPuANw613e2wryRJU2hULz1JkkaEQSFJ6mRQSJI6GRSSpE4GhSSpk0EhjZPk4SSXJbkyyT8m2W2Kt39CktPbx6ck+ZOp3L401QwK6dHur6rnV9VzaL6j8+aZLkiaSQaF1O07DEyaluQdSda0c/n/xcD6N7TrLk9yTrvu5e29Ui5N8rUkT52B+qVf2Uh+M1saBUnm0Ez7cGa7fCTNXEkH08znvzLJi2jmGHsX8O+q6vYkT2438S3g0KqqJL8H/CnNN4SlWcWgkB7t8UkuozmT+AHw1Xb9ke3Ppe3yLjTB8Tzggqq6HaCqxu4HsBD4XDvf/47Aj6enfGlqeelJerT7q+r5wF40Zw5jYxQB3teOXzy/qvapqjM7tvM/gdOr6gDg92kmopNmHYNC2or2roF/DLy9ncr+QuA/J9kFIMmCJHsA3wBeneQp7fqxS0/z2DyF8+8izVJeepI6VNWlSa4AllXVOe0U1d9pZ+m9Fzi+nan0NODiJA/TXJo6geauahckuYsmTPaeiWOQflXOHitJ6uSlJ0lSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHX6/58O/YYvDyBZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluate(model, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env",
   "language": "python",
   "name": "env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}