Email Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8c61ca4b",
   "metadata": {},
   "source": [
    "# <div style=\"text-align: center\">Gmail Classification Models</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7e07d6a",
   "metadata": {},
   "source": [
    "**Import Libraries**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ac4346c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set_style('whitegrid')\n",
    "\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1593e06",
   "metadata": {},
   "source": [
    "**Read Excel file**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a6ffbb63",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Label</th>\n",
       "      <th>Text</th>\n",
       "      <th>Label_Number</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>spam</td>\n",
       "      <td>Why United Kingdom is best study destination_x...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>non_spam</td>\n",
       "      <td>Homeowners are looking for a tenant like you z...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>non_spam</td>\n",
       "      <td>Shop Assigned  Mi Home VM JanakpuriHigh Street...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>non_spam</td>\n",
       "      <td>Profile picture pending approval_x000D_\\nHi Ru...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>non_spam</td>\n",
       "      <td>Mahimagoyal JEE Main New Exam Dates Out_x000D_\\n</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Label                                               Text  Label_Number\n",
       "0      spam  Why United Kingdom is best study destination_x...             1\n",
       "1  non_spam  Homeowners are looking for a tenant like you z...             0\n",
       "2  non_spam  Shop Assigned  Mi Home VM JanakpuriHigh Street...             0\n",
       "3  non_spam  Profile picture pending approval_x000D_\\nHi Ru...             0\n",
       "4  non_spam   Mahimagoyal JEE Main New Exam Dates Out_x000D_\\n             0"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_excel(r'C:\\Users\\mgman\\Downloads\\All_Emails.xlsx')\n",
    "\n",
    "df.drop('Unnamed: 0', axis=1, inplace = True)\n",
    "df.columns = ['Label', 'Text', 'Label_Number']\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4bcb2fc9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(980, 3)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0ac101ca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 980 entries, 0 to 979\n",
      "Data columns (total 3 columns):\n",
      " #   Column        Non-Null Count  Dtype \n",
      "---  ------        --------------  ----- \n",
      " 0   Label         980 non-null    object\n",
      " 1   Text          980 non-null    object\n",
      " 2   Label_Number  980 non-null    int64 \n",
      "dtypes: int64(1), object(2)\n",
      "memory usage: 23.1+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c89aef4d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Label           0\n",
       "Text            0\n",
       "Label_Number    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1ffd4a69",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    938\n",
       "1     42\n",
       "Name: Label_Number, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Label_Number'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5c36f56",
   "metadata": {},
   "source": [
    "**Count Plot**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "aaf94eb6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfEAAAFxCAYAAABjgpGlAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAW+klEQVR4nO3df6yedX3/8df54bmFc07bVEQlcDoqNJRI10ptp0il2UxFshAU+XFclTFJ1qhLK2HtND0gRaAjHnSair8yYlFrHW1mTDQZxdFJR0fO5opw6lzHaMEixdVwzonc57S9vn+4nch3UA7ruXv3c3w8/jr3fd331fc56ZXn+VznnOtqqaqqCgBQnNZmDwAA/N+IOAAUSsQBoFAiDgCFEnEAKJSIA0Ch2ps9wCv1ox/9KLVardljAMBxUa/XM3/+/BfdVlzEa7Va5s6d2+wxAOC4GBwcfMltTqcDQKFEHAAKJeIAUCgRB4BCiTgAFErEAaBQIg4AhRJxACiUiANAoUQcAAol4gBQKBEHgEKJOAAUSsSBpqsO1Zs9AhyzZvw/Lu5WpMDU09Jey96bz2v2GHBMevoeOe7/ppU4ABRKxAGgUCIOAIUScQAolIgDQKFEHAAKJeIAUCgRB4BCiTgAFErEAaBQIg4AhRJxACiUiANAoUQcAAol4gBQKBEHgEKJOAAUSsQBoFAiDgCFEnEAKJSIA0ChRBwACiXiAFAoEQeAQok4ABRKxAGgUCIOAIUScQAolIgDQKFEHAAKJeIAUCgRB4BCiTgAFErEAaBQIg4AhRJxACiUiANAoUQcAAol4gBQKBEHgEKJOAAUqr0ROx0bG8uaNWvy1FNPpbW1NevWrUt7e3vWrFmTlpaWnH322bnxxhvT2tqazZs3Z9OmTWlvb8+KFSuydOnSRowEAFNOQyL+wAMP5NChQ9m0aVMefPDBfOYzn8nY2FhWrlyZxYsXp6+vL9u2bcv8+fOzcePG3HvvvanX6+nt7c0FF1yQjo6ORowFAFNKQ06nn3nmmTl8+HCOHDmS4eHhtLe359FHH82iRYuSJEuWLMmOHTuya9euLFiwIB0dHenu7k5PT092797diJEAYMppyEr85JNPzlNPPZWLL744Bw8ezF133ZWHH344LS0tSZLOzs4MDQ1leHg43d3d4+/r7OzM8PDwUfddr9czODjYiLGBJpk7d26zR4BJcbz71JCI33333Xn729+e66+/Pvv3788HP/jBjI2NjW8fGRnJtGnT0tXVlZGRkRc8/5tRfzG1Ws0BD8AJqRF9Oto3Bg05nT5t2rTxGE+fPj2HDh3Kueeem507dyZJtm/fnoULF2bevHkZGBhIvV7P0NBQ9uzZkzlz5jRiJACYchqyEr/mmmvy8Y9/PL29vRkbG8uqVavypje9KWvXrk1/f39mz56dZcuWpa2tLcuXL09vb2+qqsqqVatSq9UaMRIATDktVVVVzR7ilRgcHHQ6HaagvTef1+wR4Jj09D3SkP0erXsu9gIAhRJxACiUiANAoUQcAAol4gBQKBEHgEKJOAAUSsQBoFAiDgCFEnEAKJSIA0ChRBwACiXiAFAoEQeAQok4ABRKxAGgUCIOAIUScQAolIgDQKFEHAAKJeIAUCgRB4BCiTgAFErEAaBQIg4AhRJxACiUiANAoUQcAAol4gBQKBEHgEKJOAAUSsQBoFAiDgCFEnEAKJSIA0ChRBwACiXiAFAoEQeAQok4ABRKxAGgUCIOAIUScQAolIgDQKFEHAAKJeIAUCgRB4BCiTgAFErEAaBQIg4AhRJxACiUiANAoUQcAAol4gBQKBEHgEKJOAAUSsQBoFAiDgCFEnEAKFR7o3b8xS9+Mffff3/GxsZy9dVXZ9GiRVmzZk1aWlpy9tln58Ybb0xra2s2b96cTZs2pb29PStWrMjSpUsbNRIATCkNWYnv3Lkz//Iv/5JvfvOb2bhxY55++uncdtttWblyZb7xjW+kqqps27YtBw4cyMaNG7Np06Z89atfTX9/f0ZHRxsxEgBMOQ2J+A9/+MPMmTMnH/7wh/Onf/qnueiii/Loo49m0aJFSZIlS5Zkx44d2bVrVxYsWJCOjo50d3enp6cnu3fvbsRIADDlNOR0+sGDB/Ozn/0sd911V5588smsWLEiVVWlpaUlSdLZ2ZmhoaEMDw+nu7t7/H2dnZ0ZHh4+6r7r9XoGBwcbMTbQJHPnzm32CDApjnefGhLxGTNmZPbs2eno6Mjs2bNTq9Xy9NNPj28fGRnJtGnT0tXVlZGRkRc8/5tRfzG1Ws0BD8AJqRF9Oto3Bg05nX7++efnH/7hH1JVVX7+85/nV7/6Vd761rdm586dSZLt27dn4cKFmTdvXgYGBlKv1zM0NJQ9e/Zkzpw5jRgJAKachqzEly5dmocffjiXX355qqpKX19fTj/99Kxduzb9/f2ZPXt2li1blra2tixfvjy9vb2pqiqrVq1KrVZrxEgAMOW0VFVVNXuIV2JwcNDpdJiC9t58XrNHgGPS0/dIQ/Z7tO652AsAFErEAaBQIg4AhRJxACiUiANAoUQcAAol4gBQKBEHgEKJOAAUSsQBoFAiDgCFEnEAKJSIA0ChRBwACiXiAFAoEQeAQok4ABRKxAGgUCIOAIUScQAolIgDQKEmFPFvf/vbL3j8ta99rSHDAAAT1360jd/97ndz//33Z+fOnXnooYeSJIcPH85Pf/rTfOADHzguAwIAL+6oEb/wwgvz2te+Nr/85S9z5ZVXJklaW1tzxhlnHJfhAICXdtSIT58+PYsXL87ixYvzi1/8IvV6PcmvV+MAQHMdNeL/45Of/GQeeOCBnHrqqamqKi0tLdm0aVOjZwMAjmJCEf/Xf/3X3HfffWlt9cvsAHCimFCVZ82aNX4qHQA4MUxoJb5///4sXbo0s2bNShKn0wHgBDChiH/6059u9BwAwCs0oYhv3br1fz33kY98ZNKHAQAmbkIRP+WUU5IkVVXlsccey5EjRxo6FADw8iYU8auuuuoFjz/0oQ81ZBgAYOImFPHHH398/OMDBw5k//79DRsIAJiYCUW8r69v/ONarZY///M/b9hAAMDETCjiGzduzMGDB7Nv376cfvrpmTlzZqPnAgBexoQu9vK9730vV111Ve66665ceeWV+du//dtGzwUAvIwJrcTvvvvubNmyJZ2dnRkeHs4HP/jBXHrppY2eDQA4igmtxFtaWtLZ2Zkk6erqSq1Wa+hQAMDLm9BKvKenJ7fffnsWLlyYgYGB9PT0NHouAOBlTGglfsUVV2T69OnZsWNHtmzZkve///2NngsAeBkTivjtt9+ed77znenr68vf/M3f5Pbbb2/0XADAy5hQxNvb23PWWWclSc444wz3FQeAE8CEfiZ+2mmnpb+/P/Pnz8+uXbty6qmnNnouAOBlTGhJfdttt2XmzJl54IEHMnPmzNx2222NngsAeBkTWonXarVcc801DR4FAHgl/HAbAAol4gBQKBEHgEKJOAAUSsQBoFAiDgCFEnEAKJSIA0ChRBwACiXiAFAoEQeAQok4ABSqYRH/xS9+kXe84x3Zs2dPnnjiiVx99dXp7e3NjTfemCNHjiRJNm/enPe85z254oor8oMf/KBRowDAlNSQiI+NjaWvry+vfvWrk/z6VqYrV67MN77xjVRVlW3btuXAgQPZuHFjNm3alK9+9avp7+/P6OhoI8YBgCmpIRFfv359rrrqqpx66qlJkkcffTSLFi1KkixZsiQ7duzIrl27smDBgnR0dKS7uzs9PT3ZvXt3I8YBgClpQvcTfyW2bNmSmTNn5sILL8yXvvSlJElVVWlpaUmSdHZ2ZmhoKMPDw+nu7h5/X2dnZ4aHh192//V6PYODg5M9NtBEc+fObfYIMCmOd58mPeL33ntvWlpa8o//+I8ZHBzM6tWr81//9V/j20dGRjJt2rR0dXVlZGTkBc//ZtRfSq1Wc8ADcEJqRJ+O9o3BpJ9O//rXv5577rknGzduzNy5c7N+/fosWbIkO3fuTJJs3749CxcuzLx58zIwMJB6vZ6hoaHs2bMnc+bMmexxAGDKmvSV+ItZvXp11q5dm/7+/syePTvLli1LW1tbli9fnt7e3lRVlVWrVqVWqx2PcQBgSmipqqpq9hCvxODgoNPpMAXtvfm8Zo8Ax6Sn75GG7Pdo3XOxFwAolIgDQKFEHAAKJeIAUCgRB4BCiTgAFErEAaBQIg4AhRJxACiUiANAoUQcAAol4gBQKBEHgEKJOAAUSsQBoFAiDgCFEnEAKJSIA0ChRBwACiXiAFAoEQeAQok4ABRKxAGgUCIOAIUScQAolIgDQKFEHAAKJeIAUCgRB4BCiTgAFErEAaBQIg4AhRJxACiUiANAoUQcAAol4gBQKBEHgEKJOAAUSsQBoFAiDgCFEnEAKJSIA0ChRBwACiXiAFAoEQeAQok4ABRKxAGgUCIOAIUScQAolIgDQKFEHAAKJeIAUCgRB4BCiTgAFErEAaBQIg4AhRJxACiUiANAoUQcAArVPtk7HBsby8c//vE89dRTGR0dzYoVK3LWWWdlzZo1aWlpydlnn50bb7wxra2t2bx5czZt2pT29vasWLEiS5cunexxAGDKmvSIf+c738mMGTNyxx135ODBg7nssstyzjnnZOXKlVm8eHH6+vqybdu2zJ8/Pxs3bsy9996ber2e3t7eXHDBBeno6JjskQBgSpr0iL/rXe/KsmXLxh+3tbXl0UcfzaJFi5IkS5YsyYMPPpjW1tYsWLAgHR0d6ejoSE9PT3bv3p158+ZN9kgAMCVNesQ7OzuTJMPDw/mzP/uzrFy5MuvXr09LS8v49qGhoQwPD6e7u/sF7xseHn7Z/dfr9QwODk722EATzZ07t9kjwKQ43n2a9Ignyf79+/PhD384vb29+cM//MPccccd49tGRkYybdq0dHV1ZWRk5AXP/2bUX0qtVnPAA3BCakSfjvaNwaT/dvqzzz6ba6+9NjfccEMuv/zyJMm5556bnTt3Jkm2b9+ehQsXZt68eRkYGEi9Xs/Q0FD27NmTOXPmTPY4ADBlTfpK/K677spzzz2XDRs2ZMOGDUmST3ziE7nlllvS39+f2bNnZ9myZWlra8vy5cvT29ubqqqyatWq1Gq1yR4HAKaslqqqqmYP8UoMDg46nQ5T0N6bz2v2CHBMevoeach+j9Y9F3sBgEKJOAAUSsQBoFAiDgCFEnEAKJSIA0ChRBwACiXiAFAoEQeAQok4ABRKxAGgUCIOAIUScQAolIgDQKFEHAAKJeIAUCgRB4BCiTgAFErEAaBQIg4AhRJxACiUiANAoUQcAAol4gBQKBEHgEKJOAAUSsQBoFAiDgCFEnEAKJSIA0ChRBwACiXiAFAoEQeAQok4ABRKxAGgUCIOAIUScQAolIgDQKFEHAAKJeIAUCgRB4BCiTgAFErEAaBQIg4AhRJxACiUiANAoUQcAAol4v+tPna42SPApPB/GX57tDd7gBNF7VVtOf+GrzV7DDhmA3d8oNkjAMeJlTgAFErEAaBQIg4AhRJxACiUiANAoUQcAAol4gBQKBEHgEKJOAAUqulXbDty5Ehuuumm/OQnP0lHR0duueWWzJo1q9ljAcAJr+kr8fvuuy+jo6P51re+leuvvz633357s0cCgCI0PeIDAwO58MILkyTz58/Pj3/84yZPBABlaPrp9OHh4XR1dY0/bmtry6FDh9Le/uKj1ev1DA4ONmSWe659S0P2C8dTo46Phnvf5mZPAMekUcdevV5/yW1Nj3hXV1dGRkbGHx85cuQlA578erUOAJwAp9Pf/OY3Z/v27UmSH/3oR5kzZ06TJwKAMrRUVVU1c4D/+e30f/u3f0tVVbn11lvzxje+sZkjAUARmh5xAOD/pumn0wGA/xsRB4BCiTgAFErEAaBQTf87ccr2+OOP5y/+4i/S3t6etra2vPe9783WrVvT2tqaAwcO5Morr8z73//+/NM//VM+//nPJ0mef/75rF+/Pq961auyatWqvOENb8iTTz6ZSy65JD/96U/z2GOP5aKLLsrHPvaxJn92cHxt2bIlDzzwQJ5//vns3bs31113Xc4555ysW7cubW1tqdVqWbduXY4cOZLrr78+r3/967Nv376cd955+eQnP/mS+73zzjvz0EMP5ciRI7nkkktyzTXXZPny5TnzzDPz+OOPp6qq3HnnnZk5c2b6+vry9NNP5+DBg1myZElWrlyZNWvWpL29PT/72c8yOjqad7/73fnBD36Q/fv3Z8OGDenp6TmOXyVeoIJjcM8991Q333xzNTo6Wu3YsaPauHFjdfHFF1f1er361a9+Vf3BH/xB9eyzz1b33HNP9fTTT1dVVVVf+MIXqg0bNlT79u2rFi9eXD333HPVM888U5133nnVwYMHq+eff75661vf2uTPDI6/e++9t7r22murqqqqxx9/vFq2bFl12WWXVY899lhVVVX1d3/3d9VHP/rRat++fdWiRYuqoaGh6tChQ9VFF11UPfPMMy+53yVLllR79+6t6vV69c1vfrOqqqr6oz/6o2rr1q1VVf36OF63bl21b9++avPmzVVVVdXzzz9fLVq0qKqqqlq9enW1YcOGqqqqau3atdX69eurqqqqz372s9Vf//VfT/rXgYmzEueYXH755fnyl7+cD33oQ+nu7s4FF1yQBQsWpKOjI0ly9tlnZ+/evXnd616XT33qUzn55JPz85//PG9+85uTJGeccUa6u7vT0dGRU045JTNmzEiStLS0NOtTgqY655xzkiRveMMbMjo6muHh4cydOzdJ8pa3vCWf/vSnkyQ9PT3jl6x+7Wtfe9RLc/b396e/vz/PPvvs+L0qkuT3fu/3kvz6olv3339/ZsyYkUceeSQPPfRQurq6Mjo6Ov7ac889N0kybdq0zJ49e/zj33wNx5+Ic0y2bduW888/Px/5yEfy3e9+N/39/ZkxY0YOHz6c0dHR/Pu//3tmzZqVFStW5L777ktXV1dWr16d6r8vTyDW8EL//zFx6qmnZvfu3TnnnHPy8MMP53d+53de9HUvZXR0NN///vfT39+fqqpyySWX5JJLLkmS/PjHP87rX//6/PM//3POOuusbNmyJd3d3bn55pvzxBNPZPPmzY7VE5yIc0ze9KY35YYbbsjnPve5tLa2Zvny5dm6dWuuu+66/PKXv8yKFSsyc+bMXHrppbniiisybdq0nHLKKXnmmWeaPToU4ZZbbsm6detSVVXa2tpy6623vqL3d3R0ZPr06bn00kszffr0XHDBBTnttNOSJFu3bs3dd9+dk046KX/5l3+ZZ599Nh/72McyMDCQk046KbNmzXKsnuBcsY1JtXPnzmzatCl33nlns0cBjmL58uW56aabXOa6cFbiAFPArl27cscdd/yv5y+++OL09vY2YSKOBytxACiUi70AQKFEHAAKJeIAUCgRh99iO3fuzKpVqybtda/0tcCxEXEAKJQ/MQNe4Pvf/36+/vWvjz/+7Gc/myR54okn8id/8ic5ePBgrr766rzvfe/LT37yk9xyyy1JkhkzZrziC5EAx0bEgRf4z//8z3zpS1/KSSedlL6+vvzwhz/M6173uoyNjeULX/hCjhw5kksvvTS///u/n7Vr1+bWW2/NWWedlW9/+9v5yle+kre97W3N/hTgt4aIAy/wmte8JqtXr05nZ2f+4z/+I/Pnz0+SzJ8/f/zGNm984xvz5JNPZs+ePeO3wBwbG8uZZ57ZrLHht5KIA+OGhobyV3/1V/n7v//7JMkf//Efj98A47HHHsuhQ4cyOjqaPXv2pKenJ2eeeWbWr1+f0047LQMDAzlw4EATp4ffPiIOv+UefPDBvOc97xl//Lu/+7u57LLLcvLJJ2fatGl55plncvrpp6dWq+W6667Lc889l49+9KOZMWNGbrrppqxevTqHDx9OknzqU59ywww4jlx2FQAK5U/MAKBQIg4AhRJxACiUiANAoUQcAAol4gBQKBEHgEKJOAAU6v8Bio68UmeG8DIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize = (8, 6))\n",
    "sns.countplot(data = df, x = 'Label');"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72048626",
   "metadata": {},
   "source": [
    "**Count no. of each word**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f977360d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      713\n",
       "1      114\n",
       "2      687\n",
       "3      107\n",
       "4        7\n",
       "      ... \n",
       "975     27\n",
       "976     28\n",
       "977    277\n",
       "978     15\n",
       "979      3\n",
       "Name: count, Length: 980, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def count_words(text):\n",
    "    words = word_tokenize(text)\n",
    "    return len(words)\n",
    "df['count']=df['Text'].apply(count_words)\n",
    "df['count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5a2afec2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Label_Number\n",
       "0    199.382729\n",
       "1    423.642857\n",
       "Name: count, dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('Label_Number')['count'].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9640bfc0",
   "metadata": {},
   "source": [
    "**Tokenization**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5d18cf35",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before cleaning:\n",
      "Wall time: 0 ns\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Label</th>\n",
       "      <th>Text</th>\n",
       "      <th>Label_Number</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>spam</td>\n",
       "      <td>Why United Kingdom is best study destination_x...</td>\n",
       "      <td>1</td>\n",
       "      <td>713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>non_spam</td>\n",
       "      <td>Homeowners are looking for a tenant like you z...</td>\n",
       "      <td>0</td>\n",
       "      <td>114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>non_spam</td>\n",
       "      <td>Shop Assigned  Mi Home VM JanakpuriHigh Street...</td>\n",
       "      <td>0</td>\n",
       "      <td>687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>non_spam</td>\n",
       "      <td>Profile picture pending approval_x000D_\\nHi Ru...</td>\n",
       "      <td>0</td>\n",
       "      <td>107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>non_spam</td>\n",
       "      <td>Mahimagoyal JEE Main New Exam Dates Out_x000D_\\n</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Label                                               Text  Label_Number  \\\n",
       "0      spam  Why United Kingdom is best study destination_x...             1   \n",
       "1  non_spam  Homeowners are looking for a tenant like you z...             0   \n",
       "2  non_spam  Shop Assigned  Mi Home VM JanakpuriHigh Street...             0   \n",
       "3  non_spam  Profile picture pending approval_x000D_\\nHi Ru...             0   \n",
       "4  non_spam   Mahimagoyal JEE Main New Exam Dates Out_x000D_\\n             0   \n",
       "\n",
       "   count  \n",
       "0    713  \n",
       "1    114  \n",
       "2    687  \n",
       "3    107  \n",
       "4      7  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "def clean_str(string, reg = RegexpTokenizer(r'[a-z]+')):\n",
    "    # Clean a string with RegexpTokenizer\n",
    "    string = string.lower()\n",
    "    tokens = reg.tokenize(string)\n",
    "    return \" \".join(tokens)\n",
    "\n",
    "print('Before cleaning:')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a83a3dec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "After cleaning:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Label</th>\n",
       "      <th>Text</th>\n",
       "      <th>Label_Number</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>spam</td>\n",
       "      <td>why united kingdom is best study destination x...</td>\n",
       "      <td>1</td>\n",
       "      <td>713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>non_spam</td>\n",
       "      <td>homeowners are looking for a tenant like you z...</td>\n",
       "      <td>0</td>\n",
       "      <td>114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>non_spam</td>\n",
       "      <td>shop assigned mi home vm janakpurihigh street ...</td>\n",
       "      <td>0</td>\n",
       "      <td>687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>non_spam</td>\n",
       "      <td>profile picture pending approval x d hi rupal ...</td>\n",
       "      <td>0</td>\n",
       "      <td>107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>non_spam</td>\n",
       "      <td>mahimagoyal jee main new exam dates out x d</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Label                                               Text  Label_Number  \\\n",
       "0      spam  why united kingdom is best study destination x...             1   \n",
       "1  non_spam  homeowners are looking for a tenant like you z...             0   \n",
       "2  non_spam  shop assigned mi home vm janakpurihigh street ...             0   \n",
       "3  non_spam  profile picture pending approval x d hi rupal ...             0   \n",
       "4  non_spam        mahimagoyal jee main new exam dates out x d             0   \n",
       "\n",
       "   count  \n",
       "0    713  \n",
       "1    114  \n",
       "2    687  \n",
       "3    107  \n",
       "4      7  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('After cleaning:')\n",
    "df['Text'] = df['Text'].apply(lambda string: clean_str(string))\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b58639b4",
   "metadata": {},
   "source": [
    "**Stemming words**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b9d3c836",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Label</th>\n",
       "      <th>Text</th>\n",
       "      <th>Label_Number</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>spam</td>\n",
       "      <td>why united kingdom is best study destination x...</td>\n",
       "      <td>1</td>\n",
       "      <td>713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>non_spam</td>\n",
       "      <td>homeowners are looking for a tenant like you z...</td>\n",
       "      <td>0</td>\n",
       "      <td>114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>non_spam</td>\n",
       "      <td>shop assigned mi home vm janakpurihigh street ...</td>\n",
       "      <td>0</td>\n",
       "      <td>687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>non_spam</td>\n",
       "      <td>profile picture pending approval x d hi rupal ...</td>\n",
       "      <td>0</td>\n",
       "      <td>107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>non_spam</td>\n",
       "      <td>mahimagoyal jee main new exam dates out x d</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Label                                               Text  Label_Number  \\\n",
       "0      spam  why united kingdom is best study destination x...             1   \n",
       "1  non_spam  homeowners are looking for a tenant like you z...             0   \n",
       "2  non_spam  shop assigned mi home vm janakpurihigh street ...             0   \n",
       "3  non_spam  profile picture pending approval x d hi rupal ...             0   \n",
       "4  non_spam        mahimagoyal jee main new exam dates out x d             0   \n",
       "\n",
       "   count  \n",
       "0    713  \n",
       "1    114  \n",
       "2    687  \n",
       "3    107  \n",
       "4      7  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.stem import PorterStemmer\n",
    "stemmer = PorterStemmer()\n",
    "def stemming (text):\n",
    "    return ''.join([stemmer.stem(word) for word in text])\n",
    "df['Text']=df['Text'].apply(stemming)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8705830a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of X: (980,)\n",
      "shape of y: (980,)\n"
     ]
    }
   ],
   "source": [
    "X = df.loc[:, 'Text']\n",
    "y = df.loc[:, 'Label_Number']\n",
    "\n",
    "print(f\"Shape of X: {X.shape}\\nshape of y: {y.shape}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10f24bd8",
   "metadata": {},
   "source": [
    "**Split into Training data and Test data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0afb3490",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d3800b83",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Data Shape: (784,)\n",
      "Test Data Shape: (196,)\n"
     ]
    }
   ],
   "source": [
    "print(f\"Training Data Shape: {X_train.shape}\\nTest Data Shape: {X_test.shape}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3ca5aa1",
   "metadata": {},
   "source": [
    "**Count Vectorization to Extract Features from Text**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "af3fe0b3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer()"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "cv=CountVectorizer()\n",
    "cv.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "431a5beb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No.of Tokens:  9246\n"
     ]
    }
   ],
   "source": [
    "print('No.of Tokens: ',len(cv.vocabulary_.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "17c92fae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "scipy.sparse.csr.csr_matrix"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dtv = cv.transform(X_train)\n",
    "type(dtv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "986d5563",
   "metadata": {},
   "outputs": [],
   "source": [
    "dtv = dtv.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "21e1329b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Observations: 784\n",
      "Tokens/Features: 9246\n"
     ]
    }
   ],
   "source": [
    "print(f\"Number of Observations: {dtv.shape[0]}\\nTokens/Features: {dtv.shape[1]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f8f4213e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, ..., 0, 0, 0], dtype=int64)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dtv[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "00088fcd",
   "metadata": {},
   "source": [
    "**Apply different models**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "45c47a88",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest        trained in 0.78 sec\n",
      "MultinomialNB        trained in 0.04 sec\n",
      "Logistic Regr.       trained in 0.08 sec\n",
      "KNN                  trained in 0.0 sec\n",
      "Decision Tree        trained in 0.91 sec\n",
      "SVM (Linear)         trained in 0.08 sec\n",
      "SVM (RBF)            trained in 0.56 sec\n",
      "Wall time: 2.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.svm import LinearSVC, SVC\n",
    "from time import perf_counter\n",
    "import warnings\n",
    "warnings.filterwarnings(action='ignore')\n",
    "models = {\n",
    "    \"Random Forest\": {\"model\":RandomForestClassifier(), \"perf\":0},\n",
    "    \"MultinomialNB\": {\"model\":MultinomialNB(), \"perf\":0},\n",
    "    \"Logistic Regr.\": {\"model\":LogisticRegression(solver='liblinear', penalty ='l2' , C = 1.0), \"perf\":0},\n",
    "    \"KNN\": {\"model\":KNeighborsClassifier(), \"perf\":0},\n",
    "    \"Decision Tree\": {\"model\":DecisionTreeClassifier(), \"perf\":0},\n",
    "    \"SVM (Linear)\": {\"model\":LinearSVC(), \"perf\":0},\n",
    "    \"SVM (RBF)\": {\"model\":SVC(), \"perf\":0}\n",
    "}\n",
    "\n",
    "for name, model in models.items():\n",
    "    start = perf_counter()\n",
    "    model['model'].fit(dtv, y_train)\n",
    "    duration = perf_counter() - start\n",
    "    duration = round(duration,2)\n",
    "    model[\"perf\"] = duration\n",
    "    print(f\"{name:20} trained in {duration} sec\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "218a65f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Observations: 196\n",
      "Tokens: 9246\n"
     ]
    }
   ],
   "source": [
    "test_dtv = cv.transform(X_test)\n",
    "test_dtv = test_dtv.toarray()\n",
    "print(f\"Number of Observations: {test_dtv.shape[0]}\\nTokens: {test_dtv.shape[1]}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a5f3c65",
   "metadata": {},
   "source": [
    "**Test Accuracy and Training Time**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "65359463",
   "metadata": {},
   "outputs": [],
   "source": [
    "models_accuracy = []\n",
    "for name, model in models.items():\n",
    "    models_accuracy.append([name, model[\"model\"].score(test_dtv, y_test),model[\"perf\"]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "55d371ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Test Accuracy</th>\n",
       "      <th>Training time (sec)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SVM (RBF)</td>\n",
       "      <td>0.974490</td>\n",
       "      <td>0.56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Logistic Regr.</td>\n",
       "      <td>0.969388</td>\n",
       "      <td>0.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Random Forest</td>\n",
       "      <td>0.964286</td>\n",
       "      <td>0.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>KNN</td>\n",
       "      <td>0.964286</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SVM (Linear)</td>\n",
       "      <td>0.964286</td>\n",
       "      <td>0.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>MultinomialNB</td>\n",
       "      <td>0.959184</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Decision Tree</td>\n",
       "      <td>0.959184</td>\n",
       "      <td>0.91</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Model  Test Accuracy  Training time (sec)\n",
       "0       SVM (RBF)       0.974490                 0.56\n",
       "1  Logistic Regr.       0.969388                 0.08\n",
       "2   Random Forest       0.964286                 0.78\n",
       "3             KNN       0.964286                 0.00\n",
       "4    SVM (Linear)       0.964286                 0.08\n",
       "5   MultinomialNB       0.959184                 0.04\n",
       "6   Decision Tree       0.959184                 0.91"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_accuracy = pd.DataFrame(models_accuracy)\n",
    "df_accuracy.columns = ['Model', 'Test Accuracy', 'Training time (sec)']\n",
    "df_accuracy.sort_values(by = 'Test Accuracy', ascending = False, inplace=True)\n",
    "df_accuracy.reset_index(drop = True, inplace=True)\n",
    "df_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "09954e73",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3sAAAFcCAYAAABx+6dEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABCUElEQVR4nO3deVhUdf//8dewjSIgUqaZioKihAuhZXaHlkoZyp1LBpqUWLaJd7nlUhqooWa7lWaZJW5omallC2ZaZG4FSqEplopaaq6AAcOc3x/+nK+kiAsjdXg+rsvrcuYz55z3OW9mzrzmnDljMQzDEAAAAADAVFwqugAAAAAAQPkj7AEAAACACRH2AAAAAMCECHsAAAAAYEKEPQAAAAAwIcIeAAAAAJgQYQ8A4HDw4EFdf/31ioyMrOhS/vVsNpvee+89x+3Fixfr+uuvd+oy/mnz+zvDMLRkyRL9+eefTlsGAOD/EPYAAA4ff/yx6tWrp+zsbG3cuLGiy/lX+/TTTzVx4sR/1TKcXfMPP/ygESNG6OTJk05bBgDg/xD2AAAOS5YsUWRkpK6//nqlpKRUdDn/aoZh/OuW4eyar8Q2AQD8H8IeAECStHnzZm3fvl233HKL7rjjDn3++ec6duyYYzw3N1eJiYm65ZZbdMMNN+jBBx/Uzp07HeOrV69Wr1691LJlS3Xo0EHvvPOOY6xJkyb6+OOPSyzvzPtGjhypJ598UrGxsWrVqpXmzZungoICTZw4UbfffruaNWumm2++WaNGjSpxVGjz5s2KjY1VaGiobr31Vj3//POOUxFvuukmFRYWOh6bl5en0NBQpaamnnP9f/nlFw0YMEA33nijbrrpJj311FM6fPhwiXo/+OAD3XfffWrRooU6d+5caiBet26dnnrqKcd0ixcvdowtXLhQHTp0UIsWLdS3b1/99ttvjrFjx45p1KhRatOmjW666SYNGDCgxDa+kGVs3LhRMTExatGihTp27KgXX3xRBQUFjukWL16su+66S82aNdPtt9+u1157TXa7/bw1n1ZcXKzJkycrPDxczZo1U1RUlFasWOEYt9vtmj59um6//XaFhoaqZ8+eWr16tSQpJydH9913nySpY8eOmjp16jnXCwBQjgwAAAzDSEhIMG655RajuLjY+PXXX42goCDjvffec4w/+OCDxp133ml89913RnZ2thEfH2/cfvvtRmFhofHDDz8YTZs2NV5++WVj586dxhdffGGEhoYaKSkphmEYRlBQkLFkyZISyzvzvhEjRhhBQUHG+++/b+zYscM4cOCAkZiYaERERBgbNmww9uzZY3zyySdG8+bNjVmzZhmGYRi7d+82WrZsaTz99NPG9u3bje+++874z3/+Y7z00kvGn3/+aYSEhBhffPGFY3mLFy82br75ZqOwsPCsdd+zZ48RFhZmDB061Ni2bZuxYcMGIyoqyrj77rsNm83mqLdNmzbGJ598YuzYscN4+umnjaZNmxo5OTlnza+goMCYM2eOERQUZBw4cMA4efKk8eGHHxpBQUHGfffdZ2RmZhqZmZlG586djdjYWMMwDMNutxv33nuv0b9/f2Pz5s3G9u3bjbFjxxpt2rQxDh8+fEHL+Pnnn40WLVoY77zzjvHbb78Z3333ndGlSxdj5MiRhmEYRlZWlhESEmJ8/vnnxt69e40vvvjCaNGihfHRRx+dc35/N3v2bKNjx47Gjz/+aOzZs8d4/fXXjeDgYGP37t2GYRjG888/b0RERBhr1qwxfvvtNyM5Odlo1qyZ8f333xs2m81ITU01goKCjIyMDCM3N7eUv0QAQHlxq+iwCQCoeIWFhfr0008VGRkpFxcXNWjQQCEhIVq4cKEeeOAB7dy5U998841mz56tNm3aSJLGjRunt956S0ePHlVycrJat26tJ598UpLUsGFDPfvss3J1db3gGmrWrKn777/fcbtly5bq0qWLWrVqJUmqW7eu5s2bp19++UXSqSNkV199tRITE+Xq6qpGjRpp/Pjx2rdvn/z8/BQeHq6lS5cqIiJC0qnvI3bt2lXu7u5nLXvevHny8fHRxIkTHeMvv/yyIiMj9c033+i2226TJPXs2dNx8ZqnnnpKixYt0ubNm3XdddeVmJ+Hh4e8vLwc63WmCRMmqEGDBpKk6Ohovfbaa5KktWvXasuWLVq/fr1j2sTERH3//fdauHChHnnkkTKXMXPmTLVv314PPvigJMnf31+JiYnq06ePBg8erD179shisahOnTqOf7NmzVLt2rXPW/Npu3btUtWqVXXdddepZs2aevzxx9WiRQv5+voqLy9Ps2fP1tSpUxUeHu5Y/tatWzVjxgy1adNG1atXlyT5+fmpWrVq51wGAKD8EPYAAFq5cqWOHj2qzp07O+6766679MILL2jjxo06dOiQJKlFixaO8Ro1amjkyJGSTp0C2a5duxLz7Nat20XVULdu3RK37777bn377bd6/vnn9dtvv2nHjh3avXu343G//PKLQkJCSgTK22+/3fH/Hj16aMiQITp+/LhOnjxZ4jTFv9u+fbuaN29eIggGBgaqRo0a+uWXXxxh73RIkyQfHx9JUlFR0QWvo8Vikb+/f4l5nD7F8ueff1ZxcbEjKJ1WUFCg7OzsC5p/VlaWdu3apRtuuMFxn/H/vyeXnZ2t8PBwtWzZUj179pS/v79uvfVWRUZGqk6dOhc0/z59+ujLL79Uu3bt1KxZM4WHh+vuu++Wt7e3Nm/erMLCQj3xxBNycfm/b4kUFRXp6quvvqD5AwDKF2EPAKCPPvpIkhQXF+e473RIWLhwoe64447zTu/mdnG7E5vNdtZ9VapUKXH7mWeeUWpqqrp376477rhDgwcP1rhx4y54mbfddpuqVavm+O5h48aNS/3pA6vVes777XZ7iQDo4eFx1mOMi7joiIuLiywWyzmnd3d3l6+vrxYuXHjWdJ6enhc0f3d3d3Xr1k0DBgw4a6xmzZqqUqWK5syZoy1btmjNmjX65ptvNG/ePA0dOvSc0/xdQECAUlNTtXbtWqWlpemTTz7Ru+++q7feestx1G7q1KklAu3p9QYAXHm8+gJAJXfw4EF9++236tOnj5YsWeL49/HHH+vWW2/VZ5995jitLzMz0zFdbm6u2rZtq40bNyowMLDEmHTqNMjHH39c0qkQkpub6xjbtWvXeWvKzc3Vhx9+qHHjxmnEiBHq1q2bGjZsqD179jjCUWBgoH7++WfZ7XbHdCkpKerRo4djmV27dlVqaqojNJamUaNG2rJlS4mjdDt27NCxY8cUGBh43lpL8/dQV5bGjRvr6NGjkk6d/ujv76+6devqlVde0YYNGy5oGY0aNVJ2drZjen9/fx0+fFiTJ09WXl6e0tLS9MYbb6h58+YaOHCgFixYoJiYGEfYL6vmuXPn6osvvlC7du00atQorVixQnXr1tWKFSvk7+8vd3d3/fHHHyWWv2zZMsfFXi52mwAALg9hDwAquY8//liGYeihhx5SUFBQiX8PPfSQCgoKlJGRoY4dOyoxMVEbN25Udna2Ro0aJW9vb7Vo0UL9+/fXhg0b9Oabb2rXrl36/PPPNXv2bHXo0EGSFBoaqoULF2rr1q366aef9Oyzz57zKNlpVqtVnp6eWrlypXbv3q2ff/5ZQ4cO1f79+x1X2Lzvvvt06NAhjR8/XtnZ2UpLS9PUqVPVvn17x3x69Oih7777TpmZmYqKiip1eX379tWJEyc0atQobd++XRs3btSwYcPUtGlTtW3b9pK26+nvpG3ZskV5eXllPr5t27YKDQ3Vk08+qY0bN+rXX3/VM888o1WrVikoKOiCljFgwABt3rxZEydOVHZ2ttavX68RI0boxIkTqlmzptzd3fXGG29o9uzZ2rNnj3788UetW7dOLVu2vKCajxw5ovHjx2vVqlXau3evVq5cqZycHLVs2VJVq1ZVv3799OKLL+rTTz/Vnj17NHv2bL3xxhuqV69eiflnZWXpxIkTF7lFAQAXrSKvDgMAqHhdunQxHn300VLH//vf/xpdunQxjh07ZowcOdK48cYbjbCwMOORRx4xdu3a5Xjcl19+aURFRRkhISFGx44djffff98xtn37duO+++4zmjVrZnTo0MH4+OOPjU6dOpW4GucDDzxQYrlr1qwxunTpYjRv3txo3769MWbMGGPSpElGp06dHI/ZsGGDER0dbTRr1sxo166d8fLLLzuunnla165djUceeaTM7ZCRkWH07dvXaN68uXHTTTcZI0eONP7880/HeFlXFP27EydOGH379jVCQkKMd9991/jwww+N4ODgEo/5+30HDx40hg0bZtx4441Gy5YtjZiYGGPDhg2l1vz3ZRiGYXzzzTfGPffcYzRr1sxo27atMXr0aOPo0aOOaT766CPHdm3btq0xZswY48SJE6XO70xFRUXGlClTjPbt2xshISFGp06djHfeeafE+CuvvOIY79y5s7Fw4cIS44MGDTJCQkKMCRMmlLpeAIDyYTEMfuEUAGBONptNt912m8aOHVvm9w4BADAbLtACADCdwsJCffXVV/rmm2/k4eFR4iqdAABUFhzZAwCYjt1uV3h4uDw8PDRlyhS1bt26oksCAOCKI+wBAAAAgAlxNU4AAAAAMCHCHgAAAACYEGEPAAAAAEyIsAcAAAAAJkTYAwAAAAATIuwBAAAAgAkR9gAAAADAhAh7AAAAAGBChD0AAAAAMCHCHgAAAACYEGEPAAAAAEyIsAcAAAAAJkTYAwAAAAATIuwBAAAAgAkR9gAAAADAhJwW9jIyMhQbG3vW/V999ZV69uyp6OhoLVy4UJJkt9s1duxYRUdHKzY2Vrt27XJWWQAAAABQKbg5Y6Zvv/22li5dqqpVq5a4v6ioSBMnTtQHH3ygqlWrqnfv3rr99tv1448/qrCwUCkpKUpPT9ekSZM0bdo0Z5QGAAAAAJWCU47s1a9fX1OnTj3r/uzsbNWvX1/Vq1eXh4eHWrVqpY0bN2rTpk0KDw+XJIWGhiozM9MZZQEAAABApeGUI3t33nmncnJyzro/NzdX3t7ejtvVqlVTbm6ucnNz5eXl5bjf1dVVNptNbm7nL++HH36QxWIpv8IBAAAA4F/EYrEoNDT0nGNOCXul8fLyUl5enuN2Xl6evL29z7rfbreXGfQkqWrVqgoODnZKrQAAAADwT5eVlVXq2BW9GmdgYKB27dqlo0ePqrCwUBs3btQNN9ygsLAwrVmzRpKUnp6uoKCgK1kWAAAAAJjOFTmyt2zZMuXn5ys6OlojR47Ugw8+KMMw1LNnT9WqVUsRERFKS0tTTEyMDMNQUlLSlSgLAAAAAEzLYhiGUdFFXKqsrCxO4wQAAABQaZ0vE/Gj6gAAAABgQoQ9AAAAADAhwh4AAAAAmBBhDwAAAABMiLAHAAAAACZE2AMAAAAAEyLsAQAAAIAJEfYAAAAAwIQIewAAAABgQoQ9AAAAADAhwh4AAAAAmBBhDwAAAABMiLAHAAAAACZE2AMAAAAAEyLsAQAAAIAJEfYAAAAAwIQIewAAAABgQoQ9AAAAADAhwh4AAAAAmBBhDwAAAABMiLAHAAAAACZE2AMAAAAAEyLsAQAAAIAJuTljpna7XQkJCdq2bZs8PDw0YcIE+fv7O8aXLFmimTNnytvbW927d1evXr1UVFSkkSNHau/evXJxcdH48eMVGBjojPIAAAAAwPSccmQvNTVVhYWFSklJ0dChQzVp0iTH2OHDh/Xqq68qOTlZc+bM0bJly5STk6PVq1fLZrNpwYIFGjhwoF555RVnlAYAAAAAlYJTwt6mTZsUHh4uSQoNDVVmZqZjLCcnR02bNpWvr69cXFzUvHlzZWRkqGHDhiouLpbdbldubq7c3Jxy0BEAAAAAKgWnJKrc3Fx5eXk5bru6uspms8nNzU3+/v7asWOHDh06pGrVqmnt2rVq0KCBPD09tXfvXt111106cuSIpk+fXuZyCgoKlJWV5YxVAAAAAIB/NaeEPS8vL+Xl5Tlu2+12x5G66tWra9SoURo0aJBq166tkJAQ1ahRQ++9955uvfVWDR06VPv379cDDzygZcuWyWq1lrocq9Wq4OBgZ6wCAAAAAPzjne/gl1NO4wwLC9OaNWskSenp6QoKCnKM2Ww2ZWRkaO7cuZo8ebJ27typsLAw+fj4yNvbW9KpQGiz2VRcXOyM8gAAAADA9JxyZC8iIkJpaWmKiYmRYRhKSkrSsmXLlJ+fr+joaLm7u6tHjx6yWq2Ki4uTn5+f+vXrp9GjR6tPnz4qKirS4MGD5enp6YzyAAAAAMD0LIZhGBVdxKXKysriNE4AAAAAldb5MhE/qg4AAAAAJkTYAwAAAAATIuwBAAAAgAkR9gAAAADAhCpV2Cso4qccKgrbHgAAALiynPLTC/9UVndXtRo+u6LLqJQ2Tbm/oksAAAAAKpVKdWQPAAAAACoLwh4AAAAAmBBhDwAAAABMiLAHAAAAACZE2AMAAAAAEyLsAQAAAIAJEfZgCoatoKJLqNTY/gAAAP88lep39mBeFjerdo9rXtFlVFr1x26p6BIAAADwNxzZAwAAAAATIuwBAAAAgAkR9gAAAADAhAh7AAAAAGBChD0AAAAAMCHCHoB/vAJ+2qFCOXP72wvobUVy9va3FRU7df44P7Y/AH56AcA/ntXNqv9M/U9Fl1FppQ1Kc9q8XaxWrW7X3mnzx/m1X7PaqfN3c3fV60OXOXUZKF38i1EVXQKACsaRPQAAAAAwIcIeAAAAAJiQU8Ke3W7X2LFjFR0drdjYWO3atavE+JIlSxQVFaU+ffpo0aJFjvvfeustRUdHq0ePHiXuBwAAAABcHKd8Zy81NVWFhYVKSUlRenq6Jk2apGnTpkmSDh8+rFdffVUfffSRfHx81K9fP7Vt21Z79+7Vjz/+qPnz5+vkyZN69913nVEaAAAAAFQKTgl7mzZtUnh4uCQpNDRUmZmZjrGcnBw1bdpUvr6+kqTmzZsrIyNDW7duVVBQkAYOHKjc3Fw99dRTzigNAAAAACoFp4S93NxceXl5OW67urrKZrPJzc1N/v7+2rFjhw4dOqRq1app7dq1atCggY4cOaJ9+/Zp+vTpysnJ0WOPPabPPvtMFoul1OUUFBQoKyvrgusKDg6+rPXC5bmYXl0selvx6K+5Oau/9Lbi8dw1N2f2F8A/n1PCnpeXl/Ly8hy37Xa73NxOLap69eoaNWqUBg0apNq1ayskJEQ1atSQr6+vAgIC5OHhoYCAAFmtVh0+fFhXXXVVqcuxWq3sSP5F6JW50V9zo7/mRW/NzZn9tRUWys3Dw2nzR+mcve3ttmK5uLk6bf44v4vd/uf7UMcpYS8sLEyrVq1SZGSk0tPTFRQU5Biz2WzKyMjQ3LlzZbPZFBcXp8GDB8vV1VWzZ89WXFycDhw4oJMnTzpO9QQAAMA/i5uHh57re09Fl1EpPT3nA6fO38XNVVnPfeXUZaB0wU93KLd5OSXsRUREKC0tTTExMTIMQ0lJSVq2bJny8/MVHR0td3d39ejRQ1arVXFxcfLz89Ptt9+uDRs26J577pFhGBo7dqxcXflEAQAAAAAuhVPCnouLi8aNG1fivsDAQMf/4+PjFR8ff9Z0XJQFAAAAAMoHP6oOAAAAACZE2AMAAAAAEyLsAQAAAIAJEfYAAAAAwIQIewAAAABgQoQ9AAAAADAhwh4AAAAAmBBhDwAAAABMiLAHAAAAACZE2AMAAAAAEyLsAQAAAIAJEfYAAAAAwIQIewAAAABgQoQ9AAAAADAhwh4AAAAAmBBhDwAAAABMiLAHAAAAACZE2AMAAAAAEyLsAQAAAIAJEfYAAAAAwIQIewAAAABgQoQ9AAAAADAhwh4AAAAAmJBTwp7dbtfYsWMVHR2t2NhY7dq1q8T4kiVLFBUVpT59+mjRokUlxv7880+1b99e2dnZzigNAAAAACoFp4S91NRUFRYWKiUlRUOHDtWkSZMcY4cPH9arr76q5ORkzZkzR8uWLVNOTo4kqaioSGPHjlWVKlWcURYAAAAAVBpOCXubNm1SeHi4JCk0NFSZmZmOsZycHDVt2lS+vr5ycXFR8+bNlZGRIUmaPHmyYmJidM011zijLAAAAACoNNycMdPc3Fx5eXk5bru6uspms8nNzU3+/v7asWOHDh06pGrVqmnt2rVq0KCBFi9eLD8/P4WHh2vGjBkXtJyCggJlZWVdcF3BwcEXvS4oPxfTq4tFbyse/TU3Z/WX3lY8nrvmRn/Ni96aW3n11ylhz8vLS3l5eY7bdrtdbm6nFlW9enWNGjVKgwYNUu3atRUSEqIaNWpo1qxZslgsWrt2rbKysjRixAhNmzZNNWvWLHU5VquVP8Z/EXplbvTX3OivedFbc6O/5kVvze1i+nu+YOiUsBcWFqZVq1YpMjJS6enpCgoKcozZbDZlZGRo7ty5stlsiouL0+DBg9WpUyfHY2JjY5WQkHDeoAcAAAAAKJ1Twl5ERITS0tIUExMjwzCUlJSkZcuWKT8/X9HR0XJ3d1ePHj1ktVoVFxcnPz8/Z5QBAAAAAJWWU8Kei4uLxo0bV+K+wMBAx//j4+MVHx9f6vTJycnOKAsAAAAAKg1+VB0AAAAATIiwBwAAAAAmRNgDAAAAABMi7AEAAACACRH2AAAAAMCECHsAAAAAYEKEPQAAAAAwIcIeAAAAAJhQmWHvkUceUWpqqoqLi69EPQAAAACAclBm2Hvqqaf0ww8/qEePHpoyZYp+++23K1AWAAAAAOBylBn2AgMD9dRTT2nWrFn6/fff1bVrV8XFxWnLli1Xoj4AAAAAwCVwK+sBq1ev1kcffaSdO3cqKipKo0ePls1m04ABA7R06dIrUSMAAAAA4CKVGfaWLl2qPn366Kabbipxf3x8vNOKAgAAAABcnjJP44yNjdXWrVslSUOHDtVPP/0kSbrjjjucWxkAAAAA4JKVGfYmTJigW265RZL05JNPKikpyelFAQAAAAAuT5lhz83NTY0aNZIk1atXTy4u/DQfAAAAAPzTlfmdvTp16uill15SaGioNm/erGuuueZK1AUAAAAAuAxlHqabOHGi/Pz8tHr1al111VWaOHHilagLAAAAAHAZygx7rq6uatmypbp27aomTZroiy++uBJ1AQAAAAAuQ5mnccbHx6uoqEgHDhxQcXGxrrnmGnXt2vVK1AYAAAAAuERlHtnLzc3VzJkz1aJFCy1evFgFBQVXoi4AAAAAwGW4oNM4JenkyZOqUqWKioqKnF4UAAAAAODylBn27rjjDr3xxhtq2rSp7r33Xnl5eV2JugAAAAAAl6HM7+wFBgaqTZs2slgsat++vfz9/cucqd1uV0JCgrZt2yYPDw9NmDChxHRLlizRzJkz5e3tre7du6tXr14qKirS6NGjtXfvXhUWFuqxxx5Tx44dL2/tAAAAAKCSKjPsTZ06VTfffLMkqUmTJhc009TUVBUWFiolJUXp6emaNGmSpk2bJkk6fPiwXn31VX300Ufy8fFRv3791LZtW61bt06+vr6aMmWKjhw5ou7duxP2AAAAAOASlRn2LBaLBg4cqIYNG8rF5dRZn0OGDDnvNJs2bVJ4eLgkKTQ0VJmZmY6xnJwcNW3aVL6+vpKk5s2bKyMjQ507d9add97peNzp7woCAAAAAC5emWGvZ8+eFz3T3NzcEt/tc3V1lc1mk5ubm/z9/bVjxw4dOnRI1apV09q1a9WgQQNVq1bNMe3//vc/Pfnkk2Uup6CgQFlZWRdcV3Bw8EWvC8rPxfTqYtHbikd/zc1Z/aW3FY/nrrnRX/Oit+ZWXv0tM+zVrVv3omfq5eWlvLw8x2273S43t1OLql69ukaNGqVBgwapdu3aCgkJUY0aNSRJ+/fv18CBA9WnTx9FRUWVuRyr1cof478IvTI3+mtu9Ne86K250V/zorfmdjH9PV8wLDPszZ8/X5JkGIZ27Nih6667TjfeeON5pwkLC9OqVasUGRmp9PR0BQUFOcZsNpsyMjI0d+5c2Ww2xcXFafDgwTp06JD69++vsWPHqm3bthe6bgAAAACAcygz7L300kuO/xcWFl7Q6ZURERFKS0tTTEyMDMNQUlKSli1bpvz8fEVHR8vd3V09evSQ1WpVXFyc/Pz8NGHCBB0/flxvvvmm3nzzTUnS22+/rSpVqlz62gEAAABAJVVm2DtTcXGx9uzZU+bjXFxcNG7cuBL3BQYGOv4fHx+v+Pj4EuPPPPOMnnnmmYspBwAAAABQijLD3q233ur4v81m0/333+/UggAAAAAAl6/MsPftt98qPz9fnp6e+uOPP1SrVq0rURcAAAAA4DK4lPWA119/Xa+99pok6bnnntOMGTOcXhQAAAAA4PKUGfa++uorjRw5UpL02muv6auvvnJ6UQAAAACAy1Nm2LNYLCosLJQkFRUVyTAMpxcFAAAAALg8ZX5nLyYmRlFRUQoKCtLOnTs1YMCAK1EXAAAAAOAylBn2evXqpVtvvVUHDhxQzZo1VadOnStRFwAAAADgMlzQBVref/99tWzZUpMmTeICLQAAAADwL8AFWgAAAADAhLhACwAAAACYEBdoAQAAAAATuqALtHTs2FF79uxRvXr1tHv37itRFwAAAADgMpQZ9iTJy8tLO3bsUGJiogoLC7V8+XJn1wUAAAAAuAznDXs5OTmaO3euVqxYIcMw9PLLLyssLOxK1QYAAAAAuESlXqDlscce04gRIxQQEKDly5ercePGBD0AAAAA+JcoNewZhiE3Nzf99ddfstvtslgsV7IuAAAAAMBlKDXsTZ8+XZMnT9axY8fUq1cvbd26VWvWrJHdbr+S9QEAAAAALsF5f2evdu3aio+P12effabnnntOH3zwgTp06HClagMAAAAAXKILuhqnxWJRu3bt1K5dOx0+fNjZNQEAAAAALtN5j+ydi5+fnzPqAAAAAACUozLD3pYtW0rcXr9+vdOKAQAAAACUj1JP49y4caN27Nih9957T3FxcZKk4uJizZs3jx9VBwAAAIB/uFLDno+Pjw4dOqTCwkIdPHhQ0qnv7g0fPvyKFQcAAAAAuDSlhr2goCAFBQWpV69eqlWrliRp//79uvbaa8ucqd1uV0JCgrZt2yYPDw9NmDBB/v7+jvElS5Zo5syZ8vb2Vvfu3dWrV68ypwEAAAAAXLgyr8b5+eefq0qVKjp+/LgWL16s8PBwjRo16rzTpKamqrCwUCkpKUpPT9ekSZM0bdo0SdLhw4f16quv6qOPPpKPj4/69euntm3b6ueffy51GgAAAADAxSnzAi2ffPKJunXrpjVr1uiTTz5RVlZWmTPdtGmTwsPDJUmhoaHKzMx0jOXk5Khp06by9fWVi4uLmjdvroyMjPNOAwAAAAC4OGUe2bNYLDp48KCuvvpqWSwWHTt2rMyZ5ubmysvLy3Hb1dVVNptNbm5u8vf3144dO3To0CFVq1ZNa9euVYMGDc47TWkKCgouKHyeFhwcfMGPRfm7mF5dLHpb8eivuTmrv/S24vHcNTf6a1701tzKq79lhr02bdqob9++evHFF5WUlKQ77rijzJl6eXkpLy/PcdtutztCW/Xq1TVq1CgNGjRItWvXVkhIiGrUqHHeaUpjtVr5Y/wXoVfmRn/Njf6aF701N/prXvTW3C6mv+cLhmWexjl48GCtWrVKgYGBGjZsmAYOHFjmAsPCwrRmzRpJUnp6uoKCghxjNptNGRkZmjt3riZPnqydO3cqLCzsvNMAAAAAAC5OmUf2NmzYoMTERBUXF6tz586qU6eOevXqdd5pIiIilJaWppiYGBmGoaSkJC1btkz5+fmKjo6Wu7u7evToIavVqri4OPn5+Z1zGgAAAADApSkz7L3yyiuaM2eOBg0apEcffVS9e/cuM+y5uLho3LhxJe4LDAx0/D8+Pl7x8fFlTgMAAAAAuDRlnsbp4uIiX19fWSwWWa1WVatW7UrUBQAAAAC4DGWGvfr16+vFF1/U0aNHNWPGDNWpU+dK1AUAAAAAuAylhr0nn3xSkpSYmKg6deqoVatWqlq1qsaPH3+lagMAAAAAXKJSv7N3+PDhUw9wc1Pv3r2vWEEAAAAAgMtXatjbs2ePXnrppXOODRkyxGkFAQAAAAAuX6lhr0qVKmrYsOGVrAUAAAAAUE5KDXtXX321unfvfiVrAQAAAACUk1Iv0NKsWbMrWQcAAAAAoByVGvZGjBhxJesAAAAAAJSjMn9nDwAAAADw70PYAwAAAAATIuwBAAAAgAkR9gAAAADAhAh7AAAAAGBChD0AAAAAMCHCHgAAAACYEGEPAAAAAEyIsAcAAAAAJkTYAwAAAAATIuwBAAAAgAkR9gAAAADAhAh7AAAAAGBChD0AAAAAMCE3Z8zUbrcrISFB27Ztk4eHhyZMmCB/f3/H+NKlSzVr1iy5uLioZ8+e6tOnj4qKijRy5Ejt3btXLi4uGj9+vAIDA51RHgAAAACYnlOO7KWmpqqwsFApKSkaOnSoJk2aVGL8+eef16xZszR//nzNmjVLx44d0+rVq2Wz2bRgwQINHDhQr7zyijNKAwAAAIBKwSlH9jZt2qTw8HBJUmhoqDIzM0uMN2nSRCdOnJCbm5sMw5DFYlHDhg1VXFwsu92u3Nxcubk5pTQAAAAAqBSckqhyc3Pl5eXluO3q6iqbzeYIcI0bN1bPnj1VtWpVRUREyMfHR3l5edq7d6/uuusuHTlyRNOnTy9zOQUFBcrKyrrguoKDgy9+ZVBuLqZXF4veVjz6a27O6i+9rXg8d82N/poXvTW38uqvU8Kel5eX8vLyHLftdrsj6G3dulVff/21Vq5cKU9PTw0fPlwrVqxQenq6br31Vg0dOlT79+/XAw88oGXLlslqtZa6HKvVyh/jvwi9Mjf6a27017zorbnRX/Oit+Z2Mf09XzB0ynf2wsLCtGbNGklSenq6goKCHGPe3t6qUqWKrFarXF1d5efnp+PHj8vHx0fe3t6SpOrVq8tms6m4uNgZ5QEAAACA6TnlyF5ERITS0tIUExMjwzCUlJSkZcuWKT8/X9HR0YqOjlafPn3k7u6u+vXrq3v37ioqKtLo0aMdV+YcPHiwPD09nVEeAAAAAJieU8Kei4uLxo0bV+K+M39GoXfv3urdu3eJcQ8PD7366qvOKAcAAAAAKh1+VB0AAAAATIiwBwAAAAAmRNgDAAAAABMi7AEAAACACRH2AAAAAMCECHsAAAAAYEKEPQAAAAAwIcIeAAAAAJgQYQ8AAAAATIiwBwAAAAAmRNgDAAAAABMi7AEAAACACRH2AAAAAMCECHsAAAAAYEKEPQAAAAAwIcIeAAAAAJgQYQ8AAAAATIiwBwAAAAAmRNgDAAAAABMi7AEAAACACRH2AAAAAMCECHsAAAAAYEKEPQAAAAAwIaeEPbvdrrFjxyo6OlqxsbHatWtXifGlS5eqe/fu6tmzp+bNm+e4/6233lJ0dLR69OihRYsWOaM0AAAAAKgU3Jwx09TUVBUWFiolJUXp6emaNGmSpk2b5hh//vnntXz5cnl6eqpLly7q0qWLtm7dqh9//FHz58/XyZMn9e677zqjNAAAAACoFJwS9jZt2qTw8HBJUmhoqDIzM0uMN2nSRCdOnJCbm5sMw5DFYtG3336roKAgDRw4ULm5uXrqqaecURoAAAAAVApOCXu5ubny8vJy3HZ1dZXNZpOb26nFNW7cWD179lTVqlUVEREhHx8fHTlyRPv27dP06dOVk5Ojxx57TJ999pksFkupyykoKFBWVtYF1xUcHHzpK4XLdjG9ulj0tuLRX3NzVn/pbcXjuWtu9Ne86K25lVd/nRL2vLy8lJeX57htt9sdQW/r1q36+uuvtXLlSnl6emr48OFasWKFfH19FRAQIA8PDwUEBMhqterw4cO66qqrSl2O1Wrlj/FfhF6ZG/01N/prXvTW3OivedFbc7uY/p4vGDrlAi1hYWFas2aNJCk9PV1BQUGOMW9vb1WpUkVWq1Wurq7y8/PT8ePH1apVK33zzTcyDEN//PGHTp48KV9fX2eUBwAAAACm55QjexEREUpLS1NMTIwMw1BSUpKWLVum/Px8RUdHKzo6Wn369JG7u7vq16+v7t27y8PDQxs2bNA999wjwzA0duxYubq6OqM8AAAAADA9p4Q9FxcXjRs3rsR9gYGBjv/37t1bvXv3Pms6LsoCAAAAAOWDH1UHAAAAABMi7AEAAACACRH2AAAAAMCECHsAAAAAYEKEPQAAAAAwIcIeAAAAAJgQYQ8AAAAATIiwBwAAAAAmRNgDAAAAABMi7AEAAACACRH2AAAAAMCECHsAAAAAYEKEPQAAAAAwIcIeAAAAAJgQYQ8AAAAATIiwBwAAAAAmRNgDAAAAABMi7AEAAACACRH2AAAAAMCECHsAAAAAYEKEPQAAAAAwIcIeAAAAAJgQYQ8AAAAATMgpYc9ut2vs2LGKjo5WbGysdu3aVWJ86dKl6t69u3r27Kl58+aVGPvzzz/Vvn17ZWdnO6M0AAAAAKgU3Jwx09TUVBUWFiolJUXp6emaNGmSpk2b5hh//vnntXz5cnl6eqpLly7q0qWLqlevrqKiIo0dO1ZVqlRxRlkAAAAAUGk45cjepk2bFB4eLkkKDQ1VZmZmifEmTZroxIkTKiwslGEYslgskqTJkycrJiZG11xzjTPKAgAAAIBKwylhLzc3V15eXo7brq6ustlsjtuNGzdWz5491aVLF912223y8fHR4sWL5efn5wiJAAAAAIBL55TTOL28vJSXl+e4bbfb5eZ2alFbt27V119/rZUrV8rT01PDhw/XihUr9OGHH8pisWjt2rXKysrSiBEjNG3aNNWsWbPU5RQUFCgrK+uC6woODr70lcJlu5heXSx6W/Hor7k5q7/0tuLx3DU3+mte9Nbcyqu/Tgl7YWFhWrVqlSIjI5Wenq6goCDHmLe3t6pUqSKr1SpXV1f5+fnp+PHjmjt3ruMxsbGxSkhIOG/QkySr1cof478IvTI3+mtu9Ne86K250V/zorfmdjH9PV8wdErYi4iIUFpammJiYmQYhpKSkrRs2TLl5+crOjpa0dHR6tOnj9zd3VW/fn11797dGWUAAAAAQKXllLDn4uKicePGlbgvMDDQ8f/evXurd+/epU6fnJzsjLIAAAAAoNLgR9UBAAAAwIQIewAAAABgQoQ9AAAAADAhwh4AAAAAmBBhDwAAAABMiLAHAAAAACZE2AMAAAAAEyLsAQAAAIAJEfYAAAAAwIQIewAAAABgQoQ9AAAAADAhwh4AAAAAmBBhDwAAAABMiLAHAAAAACZE2AMAAAAAEyLsAQAAAIAJEfYAAAAAwIQIewAAAABgQoQ9AAAAADAhi2EYRkUXcanS09NltVorugwAAAAAqBAFBQUKDQ0959i/OuwBAAAAAM6N0zgBAAAAwIQIewAAAABgQoQ9AAAAADAhwh4AAAAAmBBhDwAAAABMiLBXTmbMmKF+/fqpf//+evDBB5WZmandu3erQ4cOOvOCp0VFRerQoYNOnDihJk2a6Nlnny0xnwkTJqhDhw7nXMaECRP0+++/a+rUqbrzzjsVGxur2NhYRUVFadq0aZKkxYsX67bbblNsbKz69Omjfv366cCBA5KkkSNHKioqyjFdbGys9u3bp/nz52vt2rVO2jL/LuvWrdPgwYMvax4zZszQ5s2bSx2fM2eOJGnNmjVKSUm5oJratm3r6FmPHj30v//9T4WFhZdVp9k5a7sNHjxY69atK5caz3y+nv63cuXKcpn3mTZs2KCtW7eW+3zN7u+vB5999pm6du2q+++/X/Hx8SUe+5///EfSqZ526NBBubm5jrHy/JupzK70fnb+/Pklxg4ePKiEhIRyXy9JlXY/vG7dOjVp0kSffvppifujoqI0cuTIc06zePFivfDCC5KklJQUFRUVKSsrS6+//nq511fWfvr030lOTo5CQkKUmZnpGJs/f76mTp0qSerQoYPuu+8+9e3bVz169NDcuXPLvdZ/ojP3w3379lVMTMxZvb4Qzz33nPbt23fOsQt9L1Wad955R7Gxsbr77rtLvGcoLi6+5Hn+07hVdAFmsGPHDn311VeaP3++LBaLsrKyNGLECC1dulT169fX+vXr1aZNG0nSV199pTZt2sjb21u+vr7asGGDbDab3NzcVFxcXOKF4kzp6elyc3NT7dq1JUn9+vVT7969JUmFhYWKjIzUvffeK0nq2rWrhg0bJunUC+H06dM1duxYSdLw4cPVrl27EvPu1auX4uLidNNNN8nV1bX8N1Al8/DDD593fNq0aerbt+9ZfTifm2++WS+//LLj9tChQ/XVV1+pc+fOl1xnZfBv2G5nPl+d5cMPP1RkZKSaNm3q1OWY2SeffKKZM2fqvffe0wsvvKDVq1dryZIl6tat21mPPXnypJKSkpSUlHTlCzWpitjP/l3NmjWdFvYq8344ICBAy5cvV2RkpCRp27ZtOnny5AVN+9Zbb6lbt24KDg5WcHBwudd2MftpLy8vjRo1Sh9++KE8PDzOGn/33XdltVod79k6d+6sq666qjzL/Uc6cz+cl5en2NhYNWzY8KL69fTTT5c6djE9OpeHHnpIDz30kNatW6cFCxaUeM9gFoS9cuDn56d9+/bpgw8+ULt27RQcHKwPPvhAknTvvfdqyZIljp3Qhx9+qMcff1yS5ObmpptuuklpaWlq3769vv32W7Vt21Yff/zxWctITk5WXFzcOZd/5MgR2Wy2c/7A/LFjx3Tdddedt343NzeFhITo66+/VseOHS9q3SuLtLQ0vfLKK7JarfL19VVSUpK8vb2VmJiozMxMXX311dq7d6+mTZum119/XZGRkapXr55GjRolNzc3ubq66vnnn9fixYt17NgxJSQkqEWLFtq5c6eGDRumN998U6mpqSouLlbv3r0VExNTai2FhYU6cOCAqlevLkl68cUXtWHDBhmGoX79+umuu+7S5s2blZiYqGrVqumqq66S1WpVfHy8HnvsMfn6+qpdu3YaMGDAldp8/whnbrfi4mKNHTtWv//+u44cOaJ27drpySef1MiRI+Xh4aG9e/fqwIEDmjRpkkJCQjR37lwtWrRINWvW1J9//inp1NGD0aNHa8+ePSouLlZcXJwiIyMVGxurJk2aaPv27fL09FTr1q317bff6vjx43r33XcdfTuf48ePa/jw4crNzVVxcbGeeOIJtW3bVl27dlWDBg3k4eGhxMREPf300zpy5Igk6ZlnnlGTJk00cuRI7d69WwUFBXrwwQdVv359ffPNN/rpp5/UqFEj1alTx6nb2YyWLFmiOXPmaNasWY7+DR06VFOnTtXNN998Vjjo1q2bfvzxR61atUq33357RZRsOhW9n5WknJwcDRkyRAsXLlRUVJRuuukmbdu2TRaLRW+++aa8vb3P+Xq8fv16x1Gnv/76S5MnT5a7u/tZr8eVdT/ctGlT/fbbbzp+/Lh8fHy0dOlSRUVFaf/+/frPf/6jtLQ0SaeOkJ+5b1y0aJEOHjyowYMH64EHHnC8Ub/jjjsUFhamX3/9VVdddZWmTp0qu91+Sa/XK1eudOynX3zxRWVmZiovL0+BgYGaOHFiifXw9/dX69at9fLLL2vEiBGlru9ff/0lq9Uqb29v52zQf7Bq1aopOjpan332mYKDg8/5fMnIyNBzzz0nwzBUq1YtvfDCCxowYIASEhJ09OhRTZ48WW5ubvLx8dELL7ygL774wtGjd999V5988onc3NzUunVrDR8+XFOnTlVOTo7+/PNP7du3T6NGjVJ4eHiZtU6dOlU//vij8vPz9dxzz+m7777T8uXLZbFYFBkZqfvvv1/79+/XmDFjVFBQIKvVqvHjx+vaa6+9Alvy4nAaZznw8/PTtGnT9MMPPyg6OlqdO3fWqlWrJEmdOnXShg0b9Ndff+nAgQM6dOhQiV+479q1q+OQ9vLlyxUVFXXOZaxfv15BQUGO2++995769u2rjh07avDgwZowYYK8vLwc8zl92trMmTNLfOoxZcoUxyHq06d+SlKTJk20fv36ctsmZmIYhsaMGaPXX39dc+bM0Y033qhp06Zp5cqVOnr0qD744AMlJSVp//79Jab77rvvFBISolmzZunRRx/VsWPH9Nhjj6l69eolPh3++eeftWbNGi1atEgLFizQjh07SpySJEnff/+9YmNjFRkZqR49eigiIkJt27bV6tWrlZOTowULFmj27NmaPn26jh8/rmeffVaTJk3S7NmzVb9+fcd8Dh48qJkzZ1aaoFfadtu/f79CQ0M1c+ZMzZ8/v8TpWnXq1NHMmTMVGxurlJQUnThxQrNnz9bChQv15ptvqqioSNKpo+Y1atTQggULNGvWLL3yyis6fPiwJKlFixZ6//33VVhYqCpVqmjWrFlq1KiRNmzYcFaNp5+vsbGx+t///ifp1NHfW265RXPnztWrr76qp59+Wna7Xfn5+Xr88cf10ksvafr06br55puVnJys8ePHKyEhQbm5uVq3bp1ef/11vf322youLlazZs0UHh6u4cOHE/QuwcaNG7Vw4UIdO3asxGk911xzjZ544olzfuLs6uqqSZMmKSkpyRHGcXkqYj97Pnl5eerSpYvmzJmja665RmvWrCn19Xj79u2aMmWKZs+erQ4dOuizzz6TdPbrcWXeD0dEROjLL7+UYRjavHmzbrjhhjKn6dWrl2rWrHnWkZg9e/boiSeeUEpKig4fPqwtW7Zc9ut1bm6ufHx8NGvWLC1YsEDp6en6448/zqrpySefVFpamjZu3HjWWP/+/dW3b1917txZrVq1kru7+8VuJlO46qqrdOTIkVKfL2PGjNHEiRO1aNEitW3bVtnZ2Y5pU1NTFRERoTlz5uiee+7R8ePHHWPbtm3TihUrtGDBAi1YsEC7du1yvEZ4eHjonXfe0dNPP6333nvvgmsNCAjQggULZBiGPv30U82bN0/z5s1Tamqqdu7cqcmTJys2NlbJycl68MEHHacX/9NwZK8c7Nq1S15eXo5PebZs2aKHH35Ybdq0ka+vrzp16qTU1FTt27dPPXv2LDFtq1atlJiYqCNHjujo0aOlHoWz2+0lTgs4fRpnZmamhgwZogYNGjjGzjwtbO3atXr88cf15ZdfSjr3aZzSqdNTvv/++8vaDmZ15MgReXl5qVatWpKkG2+8US+99JJq1KjheEPh5+engICAEtPdc889evvtt/XQQw/J29u71O8C/vrrr2rRooVcXV1VtWpVPfPMM2c95vRpEEeOHFH//v1Vt25dSdIvv/yin376SbGxsZIkm82mffv26cCBA2rcuLGkU39jp9/o1K1b95ynl5hVadvN19dXW7Zs0ffffy8vL68S3+M7fWpJ7dq19cMPP2jnzp1q1KiRY7u1aNFCkpSdna1bbrlF0qnTdwIDA7Vnzx5JUkhIiCTJx8dHjRo1cvy/oKDgrBrPdRpndna24w1prVq15OXl5Xhj0rBhQ0mnev/9999rxYoVkk4dDfTy8tKYMWM0ZswY5ebm6r///e9lbT+cem2cNWuWFi1apOHDh+vtt992jP33v/9Vamqq5s2bd9Z0DRo00P3336/ExERZLJYrWbIpVcR+tizXX3+9JOnaa69VQUGB9u3bd87X41q1aum5556Tp6en/vjjD4WFhUk6+/W4Mu+Ho6KilJCQoHr16ql169bnfMzfPwQtTY0aNRxHV0735nJfr61Wqw4fPqwhQ4bI09NT+fn5jg/+zuTh4aGJEydq6NChjq/WnHbmaZwPP/ywli5dqrvvvvuC1slM9u3bp9q1a5f6/uXPP/9UYGCgJOm+++4rMe2jjz6q6dOn64EHHlCtWrUc+2NJ2rlzp1q2bOkI0a1bt9b27dslldyvX8z39s/c3+7bt0/9+vWTdOqsud27d+uXX37RW2+9pXfeeUeGYfxjAzxH9srBtm3blJCQ4HhhaNiwoby9vR3n3ffq1UvLly9XamrqWW++LBaL2rdvr4SEBHXq1KnUZVit1nN+WbRZs2YaMGCAhgwZIrvdftb4tddee84XpL87fvy4/Pz8ynxcZVSjRg3l5uY6LnSzfv16NWjQQI0bN1Z6erqkU0/83377rcR0K1euVKtWrfT++++rc+fOeueddySdvcMKCAjQzz//LLvdrqKiIsXFxZX6YlSjRg1NmTJFzzzzjA4cOKCAgAC1adNGycnJev/993XXXXepbt26ql27tnbs2CFJysjIcEzv4lI5n/J/326LFy92nHLVv39//fXXX46+/P2Neb169bRjxw799ddfKi4uVlZWliQpMDDQ8eltbm6ufvnlF0eYvFxnzvuPP/7Q8ePH5evrK+n/ehgQEKB+/fopOTlZr7zyiqKionTgwAH99NNPeuONNzRjxgxNmTJFNptNFovlgt8ooSR/f39ZrVb17dtX7u7uJc6IkKSEhAS9++67ysvLO2vavn376ujRo5X2DXx5qsj9bGn+/lpR2uvxM888o6SkJE2aNEnXXHON47n499fjyrwfrlevnvLz85WcnFyifzabTXl5eSosLHTs085ksVjOeu9zrg9XLvf1es2aNdq/f79eeuklDRkypMQ+4+9CQkLUtWvXEh8MncnDw0NXXXXVBb03M5vc3FwtWrRInTt3LvX5cs011zjeT82YMcNxsEKSli1bpu7duys5OVmNGzfWwoULHWMBAQHavHmzbDabDMPQhg0bHGHtUj9wO3N/26hRI82ePVvJycnq0aOHgoKCFBAQoGHDhik5OVmJiYm68847L3HLOBdH9srBHXfcoezsbPXq1Uuenp4yDENPPfWU43zswMBA5efnKzAw8JznaEdFRalnz54aN25cqcsICwvTTz/9VOJTjNN69eqlFStWaP78+apataqWL1+ujIwMubq6Ki8vT4mJiWWuQ0ZGhuNqcpVdWlqaevTo4bj94osvasKECRo0aJAsFouqV6+uiRMnqkaNGlqzZo1iYmJ09dVXq0qVKiU+1WnWrJnjfHEXFxeNGjVK0qm/h2HDhjk+ZQwODlZ4eLh69+4tu92u3r17n/fT5UaNGik2NlYTJkzQq6++qvXr16tPnz7Kz89Xp06d5OXlpWeffVajR4+Wp6en3N3dHUclz9S/f39Nnz690hzpO3O7DRo0SEOGDNGmTZtUtWpV+fv7O8L83/n5+emJJ55QTEyM/Pz8VLVqVUmnvic0ZswY9e7dWwUFBYqPjy+3L9s/8sgjGj16tD7//HP99ddfGjdunNzcSr5cP/roo3r66ae1cOFC5ebmKj4+XjVr1tTBgwfVrVs3eXp6qn///nJzc1PLli31wgsvqG7duo5PTHHxkpKS1K1bN7m6ujouJuHn56eRI0dq4MCBZz3eYrEoKSmp1NMGceEqYj87Y8YMLVq0SNKp7xr9/Ttaf9ehQ4dzvh7ffffduvfee+Xj46Orr7661Neayr4fjoyM1Mcff6yGDRs6jrrdf//9io6OVt26dc95Gnrr1q318MMPn/P5d6bLfb1u0aKF3nzzTd17773y8PBQvXr1Su2jdOr1+fQphKf1799fLi4ustvtql27dqU58+L01ylcXFxUXFysQYMGKSAgQA0bNjzn8yUxMVGjR4+Wi4uLatasqX79+mn27NmSpObNm2vkyJGO9zbjxo1znG7bpEkT3XXXXY73Uq1atVKnTp3K5UrUTZs2Vdu2bdW7d28VFhaqRYsWqlWrlkaMGOH4EOqvv/4674VkKpLF4OPef4Uff/xRn3zyyTlP8btcNptNcXFxeu+99yrdVcAuR3Z2trZu3aouXbroyJEj6tq1q1atWvWPCE9z587VXXfdJT8/P7388styd3c/61LxAID/48z9bFnYDwNwlsp5Tte/0A033KDi4mL9/vvv5T7vlJQUPfLII+xgLtK1116r5cuX695779VDDz2kYcOG/SOCnnTqC9D9+/dXnz59tHXr1rPOewcAlOTM/WxZ2A8DcBaO7AEAAACACXFkDwAAAABMiLAHAAAAACZE2AMAAAAAEyLsAQAgad26dWrSpIk+/fTTEvdHRUVp5MiRZU5fUFCgDh06nHf+gwcPvuw6AQC4UIQ9AAD+v4CAAC1fvtxxe9u2bTp58mQFVgQAwKXjR9UBAPj/mjZtqt9++03Hjx+Xj4+Pli5dqqioKO3fv19Lly7V+++/Lw8PDzVo0EDjxo1TYWGhhg0bpuPHj6t+/fqO+Wzbtk0TJkyQJPn6+iopKamiVgkAUIlxZA8AgDNEREToyy+/lGEY2rx5s2644QYdPXpUU6dO1fvvv6/58+fL29tbKSkp+uijjxQUFKS5c+cqJibGMY8xY8bo2WefVXJystq1a6d33nmnAtcIAFBZcWQPAIAzREVFKSEhQfXq1VPr1q0lSXa7XY0aNZKXl5ck6cYbb9S3334rSQoPD5cktWzZUm5up3ar2dnZSkxMlCQVFRWpYcOGV3o1AAAg7AEAcKZ69eopPz9fycnJGjJkiPbs2SOLxaLs7Gzl5+fL09NT69evV8OGDWWxWJSenq5OnTrp559/ls1mkyQ1bNhQkydPVp06dbRp0yYdPHiwgtcKAFAZEfYAAPibyMhIffzxx2rYsKH27NmjGjVqqGvXrrr//vvl4uKi+vXra9iwYXJ1ddWoUaPUu3dvBQQEyN3dXZKUkJCgESNGqLi4WJL03HPP6cCBAxW5SgCASshiGIZR0UUAAAAAAMoXF2gBAAAAABMi7AEAAACACRH2AAAAAMCECHsAAAAAYEKEPQAAAAAwIcIeAAAAAJgQYQ8AAAAATIiwBwAAAAAm9P8AWKADBtExeC0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1080x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize = (15,5))\n",
    "sns.barplot(x = 'Model', y ='Test Accuracy', data = df_accuracy)\n",
    "plt.title('Accuracy on the test set\\n', fontsize = 15)\n",
    "plt.ylim(0.825,1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "9dbcfece",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3UAAAFMCAYAAABs5JVsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA8EUlEQVR4nO3deVgVdf//8ddhFURxTe0WETCIXFIkzQpL01KSSk1BEzUz6+5udUlcQ1RwLbtxy0xT3FAzU1PvO5fiq7nfrrnjkkulJqZgss7vDy/PzxMgoAKOPB/X5XV5Zs75zHtmzpk5Lz6fM2MxDMMQAAAAAMCU7Iq7AAAAAADA7SPUAQAAAICJEeoAAAAAwMQIdQAAAABgYoQ6AAAAADAxQh0AAAAAmBihDgAKQUREhPz8/HL917x589tq9/Tp0/Lz89P27dvz9fwlS5bokUceua1l5dfOnTu1Y8cO62M/Pz99++23hbrMvxsxYoQaNGighg0b6sKFC0W67NsVHh6uQYMGFXcZ2UyePLlA78+IiAh17949x3kFfb8CAG6PQ3EXAAD3o0GDBqlPnz6SpF9//VUdOnTQ5MmTVa9ePUmSvb39bbVbrVo1bdiwQeXKlcvX84ODg9W0adPbWlZ+denSRcOHD1fDhg0lSRs2bFDZsmULdZk3O3LkiOLi4jRs2DA99dRTqlSpUpEtG7dW0PcrAOD2EOoAoBCUKVNGZcqUkSSlpqZKktzd3VW5cuU7atfe3r5AbZQqVUqlSpW6o2XmxTAMm8d3uo4FdfnyZUnSk08+qerVqxfpsnFrBX2/AgBuD8MvAaCYLFmyRM8//7wiIyPVsGFDffTRR5Kk+fPnq02bNqpbt64aNGigHj166OTJk5KyD2cLDw/X+PHj1a9fPwUEBKhp06YaPny4MjIyrMu4efiln5+fFi9erFdffVX16tVTq1atFB8fb1PX9OnT9cwzz+jRRx/Vv/71L40YMULh4eE5rkPz5s2VmZmpAQMGWJ9z8/DLiIgIDR48WKNHj1ZgYKAaN26sSZMm6ciRIwoLC1O9evX00ksvae/evdY2//zzTw0YMECNGzdWo0aN9MYbb+jYsWO5bsPOnTtLklq0aKGIiAhJ0uHDh/XGG2/oscceU6NGjfTRRx/p4sWLNtvhs88+U9OmTdW0aVOdP38+W9tpaWkaNWqUnnrqKQUEBKhLly7atWuXdX5WVpYmT56s5557TnXq1FFgYKDeffddm+WcOHFCb731lgICAvT4449r0KBBSklJsc5PTk7WRx99pAYNGqhx48YaOXKkMjMzc1zXLVu2qG7dukpISNDzzz+vevXqqXv37vrtt98UFRWlhg0b6oknntC0adNsXrd48WK1adNG9erVU8uWLTVnzhyb+StXrlTr1q1Vr1499ezZU0lJSTbzC7I//q6g79e/y8zM1OjRoxUUFKQ6deooJCREq1atss7PysrS1KlT1axZM9WvX1/t27fXjz/+aNPGnj17FB4ervr16+upp57SmDFjcl0eAJgVoQ4AitGJEyeUnJyspUuX6s0339Tq1asVExOjt99+W6tXr9bnn3+uM2fOaPTo0bm2MXPmTHl5eenrr7/Wm2++qblz5+q7777L9fnjxo3Tq6++qm+++UaBgYGKjIzUmTNnJElxcXGaNGmSPvzwQ33zzTf6xz/+kS0E3Gzx4sWyt7fXwIEDFRsbm+Nzli5dKul6AOvWrZv+/e9/61//+pfefPNNLVq0SI6OjoqKipJ0vdevV69eOnfunKZPn6558+bpwQcfVOfOnbOFDen68NLJkydLkhYtWqRBgwbp9OnT6tSpk9zd3TV37lxNnjxZBw8eVI8ePWwC06JFi/T5559r4sSJOfYmffTRR9q2bZsmTJigr7/+Wo8//ri6du2q48ePW7f77NmzNXjwYP3nP//R+PHjtWPHDk2ZMkXS9R7ELl26SJLmzZunzz//XDt37tTQoUOty1i9erU8PT21bNkyRUREKC4uTsuXL891e6enp+vf//63xo0bp1mzZmn//v168cUX5erqqsWLFys0NFTjx4/X0aNHrTUOHz5c3bp107Jly/T6669rzJgxmjFjhiRp27Zt6t27t9q2batvv/1WTz31lObOnWtdXkH3R34U5P06b948ff/994qNjdXq1avVqlUr9enTR6dOnZIkjR8/XkuWLFFUVJS+/fZbtW3bVu+88462bNkiSTp16pS6du0qT09PLV68WGPHjtWyZctyfa8CgGkZAIBCderUKcPX19fYtm2bzfSvv/7a8PX1NRITE63TtmzZYixfvtzmeZ999pnx7LPP5thWly5djFdeecXm+S+++KIRGRlpXYa/v791nq+vrzFmzBjr4z///NPw9fU1Vq5caRiGYTRr1syYMGGCTXvt2rUzunTpkuv6+fv7G19//bXNMpYuXWoYhmH079/fCAoKMrKysgzDMIyUlBTD19fX+Oyzz6zPnzt3rvHoo48ahmEYGzduNPz9/Y0rV67YLOO5554zpk6dmuPyt23bZvj6+hqnTp0yDMMwRo8ebTzzzDNGWlqa9TlHjx41fH19jfXr11trHDduXK7rdOLECcPX19c4fPiwzfTu3bsbQ4YMMQzDMNauXWv88MMPNvM/+ugjo2vXroZhGMb8+fONBg0a2KzLjh07jNjYWMMwru+70NBQm9eHhIQYw4cPz7GmzZs3G76+vsaGDRus0959912jWbNm1u37119/WfdnVlaW8cQTTxjjx4+3aWfMmDHG448/bmRlZRnvv/++td4b3nvvPaNZs2aGYeRvf/Tv39/o1q1bjjUX9P36d8OHDzfatGljnDt3zjAMw8jKyjISEhKMy5cvG8nJyUadOnWs+/SGQYMGGT169DAMwzDGjRtnPPvss0ZGRoZ1/rp164w5c+bkuDwAMCt+UwcAxchisdj8DqxRo0Y6fPiwJk6cqGPHjun48eM6fPiwqlSpkmsbNWvWtHlctmxZpaen5+v5Ny5okp6erqSkJJ05c0b169e3eX5AQIAOHjyY/5X6mxo1ashisUiSXF1drdNuKFWqlNLS0iRJ+/fvV2ZmpoKCgmzaSE1NVWJiYr6Wd+TIEdWtW1eOjo7WaT4+PipfvrwOHz6sZ555RpLk4eGRaxv79++XJHXs2NFmelpamrXW5s2ba+fOnfr00091/PhxHTt2TImJiQoMDJR0fQiot7e33NzcrK8PCAhQQECA9bGXl5dN++7u7rp27dot1+/mbefq6qrq1atbt++N30+mpaXp4sWLunDhgs3yJOmxxx7T9OnT9ccff+jIkSN6+umnbebXr1/fOhz2buyPvyvI+7Vz5876/vvv1bRpU9WpU0dBQUF66aWXVKZMGe3Zs0dpaWl6//33ZWf3/wcepaenWy+Wc/jwYdWuXdvmwkTNmjW7rboB4F5GqAOAYmRnZycnJyfr42+//VaDBg3Siy++qMDAQHXp0kUJCQlatmxZrm3c/PobjL9dvCQ/z3dwcMjztbfjRrs3uxFC/s7R0VHlypXTwoULs827EQjz4uzsnOP0rKwsm6CX2/Nu1CFJCxYsyHahmRvbb8qUKZo2bZratWunoKAgvfnmm5o9e7bOnj0rKef1/rubw8gNeW3/m9chtzak3NfvxhBUBwcHWSyWbMu7uf27sT/+riDvV29vb61Zs0abNm3Sxo0b9d1332nGjBn6/PPP5e7uLkmKjY2Vp6enzetubJP87AMAuB/wmzoAuIfMnj1bYWFhio6OVufOnRUQEKBffvnlrgetnJQpU0YPPvigdu/ebTN9z549t3xdbgHtdjz00EO6dOmSJMnT01Oenp6qXr26JkyYoG3btuWrjVq1amnv3r02vT9Hjx7Vn3/+KR8fn3zXIUl//PGHtQ5PT0999dVXWrt2rSRp1qxZeu+99zRkyBB16NBBtWvX1smTJ637ysfHR8ePH7e5MMr//d//6ZlnntFff/2VrzruhJubm6pWrWpzD0FJ2rFjhypXrix3d3c9/PDD2rlzp838ffv2Wf9/N/bHnZg7d67++9//qmnTphowYIBWrVql6tWra9WqVfL09JSjo6N+//13m320fPlyLVmyRNL1fbB//35lZWVZ24yPj1e7du0KvXYAKEqEOgC4h1SoUEE7duzQwYMHdeLECU2cOFErV660DvkrbD179tSsWbO0fPlyHT9+XJ988onNFR9zUrp0aR09elR//PHHHS+/SZMmql+/vj744ANt375dx48f1+DBg7V+/Xr5+vrmq40uXbroypUrGjBggI4cOaLt27erb9++evjhh9WkSZN8teHp6ang4GANGTJEP/74o3755Rd9+umnWrBggTUYVqhQQRs2bFBiYqKOHDmiqKgo7dy507qvQkJCVLp0aQ0YMECHDx/Wzp07FRMTo0aNGsnFxeX2NlAB/fOf/9Ts2bO1aNEinTx5UgsXLtScOXPUvXt3WSwWdevWTXv37rUOIV2wYIFWrlxpff3d2B93IikpScOHD9f69et15swZrV27VqdPn9ajjz4qFxcXde/eXePHj9fKlSt16tQpzZ49W5MmTbIOrX311Vd14cIFDR8+XImJidq4caNiY2OzDTkFALNjXAIA3EOGDBmiwYMHKywsTC4uLqpXr56ioqI0dOhQ67C+wnTjqoajR4/W5cuX9cwzz6hFixa3/J3XG2+8ocmTJ+unn36yXunydlksFk2aNEmjR4/W22+/rbS0NPn7+2v69OmqVatWvtqoVKmSZsyYobFjx6p9+/ZycXFR8+bN1a9fv2xDF29lxIgRGj9+vAYOHKgrV67Ix8dHsbGx1mA4evRoRUVFqW3btipbtqwaNWqkPn36aOrUqfrrr7/k6uqqL7/8UjExMerQoYNKly6t559/Xv369butbXM7wsLCdO3aNX3++ecaNmyYPDw8FBERYb0NRN26dTVlyhSNHz9eM2bMUJ06dfTaa69Zr8B5N/bHnXjrrbd07do1DRs2TBcuXFC1atX07rvvqm3btpKkDz74QI6OjhozZowuXLggDw8PRUVFWXviqlSpoi+++ELjxo3Tyy+/rAoVKuiVV17RO++8U+i1A0BRshhFMaYHAGAKCQkJ8vPzs7kwy+uvv64qVaooOjq6GCsDAAC5oacOAGC1dOlSnT59WkOHDlW5cuX0ww8/aNOmTfryyy+LuzQAAJALeuoAAFZJSUmKjo7Whg0blJKSIm9vb7355ptq3bp1cZcGAAByUWihbvfu3Ro3bpzi4uJspq9bt06TJk2Sg4OD2rdvn+0eQAAAAACA/CuU4ZdffPGFli1blu3qXunp6YqJidHixYvl4uKiTp06qVmzZqpcuXJhlAEAAAAA971CuaVBjRo1FBsbm216YmKiatSoIXd3dzk5Oalhw4bavn17YZQAAAAAACVCofTUPf/88zp9+nS26cnJySpTpoz1cenSpZWcnJxne//73//u6s1tAQAAAMBMLBaL6tevn+O8Ir36pZubm1JSUqyPU1JSbEJeblxcXOTv71+YpQEAAADAPevAgQO5ziuU4Ze58fHx0cmTJ3Xp0iWlpaVp+/btatCgQVGWAAAAAAD3lSLpqVu+fLmuXr2q0NBQRURE6PXXX5dhGGrfvr3NDW4BAAAAAAVjivvUHThwgOGXAAAAAEqsW2WiIh1+CQAAAAC4uwh1AAAAAGBihDoAAAAAMDFCHQAAAACYGKEOAAAAAEyMUAcAAAAAJkaoAwAAAAATI9QBAAAAgIkR6gAAAADAxAh1AAAAAGBihDoAAAAAMDFCHQAAAACYGKEOAAAAAEyMUAcAAAAAJkaoAwAAAAATI9QBAAAAgIkR6gAAAADAxAh1AAAAAGBihDoAAAAAMDFCHQAAAFBCZWVkFncJJdbd3PYOd60lAAAAAKZi52CvAyPXFXcZJZL/oOZ3rS166gAAAADAxAh1AAAAAGBihDoAAAAAMDFCHQAAAACYGKEOAAAAAEyMUAcAAAAAJkaoAwAAAAATI9QBAAAAgIkR6gAAAADAxAh1AAAAAGBihDoAAAAAMDFCHQAAAACYGKEOAAAAAEyMUAcAAAAAJkaoAwAAAAATI9QBAAAAgIkR6gAAAADAxAh1AAAAAGBihDoAAAAAMDFCHQAAAACYGKEOAAAAAEyMUAcAAAAAJkaoAwAAAAATI9QBAAAAgIkR6gAAAADAxAol1GVlZWno0KEKDQ1VeHi4Tp48aTN/2bJlatu2rdq3b6958+YVRgkAAAAAUCI4FEaja9asUVpamuLj47Vr1y6NGjVKU6ZMsc4fM2aMVqxYIVdXV73wwgt64YUX5O7uXhilAAAAAMB9rVBC3Y4dOxQUFCRJql+/vvbt22cz38/PT1euXJGDg4MMw5DFYimMMgAAAADgvlcooS45OVlubm7Wx/b29srIyJCDw/XFPfTQQ2rfvr1cXFzUsmVLlS1b9pbtpaam6sCBA4VRKgAAAFBi+fv7F3cJJdrdyjiFEurc3NyUkpJifZyVlWUNdAcPHtQPP/ygtWvXytXVVf369dOqVavUunXrXNtzdnbmDQcAAADgvlKQjHOrAFgoF0oJCAhQQkKCJGnXrl3y9fW1zitTpoxKlSolZ2dn2dvbq0KFCrp8+XJhlAEAAAAA971C6alr2bKlNm7cqLCwMBmGoejoaC1fvlxXr15VaGioQkND1blzZzk6OqpGjRpq27ZtYZQBAAAAAPc9i2EYRnEXkZcDBw4w/BIAAAAoBAdGrivuEkok/0HNC/T8W2Uibj4OAAAAACZGqANwz0jNSC3uEko0tj8AAOZUKL+pA4Db4ezgrCdjnyzuMkqsje9uLO4SAADAbaCnDgAAAABMjFAHAAAAACZGqAMAAAAAEyPUAQAAAICJEeoAAAAAwMQIdQAAAABgYoQ6AAAAADAxQh0AAAAAmBihDgAAAABMjFAHAAAAACZGqAMAAAAAEyPUAQAAAICJEeoAAAAAwMQIdQAAAABgYoQ6AAAAADAxQh0AAAAAmBihDgAAAABMjFAHAAAAACZGqAMAAAAAEyPUAQAAAICJEeoAAAAAwMQIdQAAAABgYoQ6AAAAADAxQh0AAAAAmBihDgAAAABMjFAHAAAAACZGqAMAAAAAEyPUAQAAAICJEeoAAAAAwMQIdQAAAABgYoQ6AAAAADAxQh0AAAAAmBihDgAAAABMjFAHAAAAACbmcKuZhmHohx9+0NatW3Xp0iVVqFBBTZo00ZNPPimLxVJUNQIAAAAAcpFrT92mTZvUrVs3bd26VX5+fmrTpo1q166tDRs2qHv37vrpp5+Ksk4AAAAAQA5y7ak7efKkZs6cKXt7e5vpwcHByszMVHx8vJ544olCLxAAAAAAkLtce+rCwsJkb2+vtLQ0HThwQJK0Zs0apaeny97eXp07dy6yIgEAAAAAOcvzQil9+/bVrl27JEnHjx9XREREYdcEAAAAAMinPEPd77//rk6dOkmS3njjDZ07d67QiwIAAAAA5E++bmlw/PhxSdIvv/yirKysQi0IAAAAAJB/t7ylgSQNHDhQH3zwgf744w898MADGjZsWFHUBQAAAADIhzxD3aOPPqo5c+bozJkz8vDwUOnSpYuiLgAAAABAPuQZ6v7zn/9oypQpyszMVKtWrWSxWPT222/f8jVZWVmKjIzUoUOH5OTkpBEjRsjT09M6f8+ePRo1apQMw1DlypU1duxYOTs73/naAAAAAEAJk+dv6mbOnKmFCxeqXLlyevvtt7VmzZo8G12zZo3S0tIUHx+vPn36aNSoUdZ5hmFoyJAhiomJ0fz58xUUFKQzZ87c2VoAAAAAQAmVZ0+dnZ2dnJycZLFYZLFY5OLikmejO3bsUFBQkCSpfv362rdvn3Xe8ePHVa5cOc2aNUuHDx/W008/LW9v7ztYBQAAAAAoufLsqQsMDFSfPn30+++/a+jQoapbt26ejSYnJ8vNzc362N7eXhkZGZKkpKQk7dy5U507d9bMmTO1efNmbdq06Q5WAQAAAABKrjx76nr37q2EhAT5+/vLx8dHzZo1y7NRNzc3paSkWB9nZWXJweH6osqVKydPT0/VqlVLkhQUFKR9+/apSZMmubaXmpqqAwcO5LlcAObm7+9f3CWUeBxrAaBk4dxbvO7WeTfPUHfu3Dk9+OCDql69uqZPn66qVavmufMDAgK0fv16BQcHa9euXfL19bXO8/DwUEpKik6ePClPT09t375dr7zyyi3bc3Z25g0HAEWAYy0AAEWnIOfdWwXAPIdf9u/fXxcuXNCECRP05JNPKjo6Os8FtmzZUk5OTgoLC1NMTIwGDBig5cuXKz4+Xk5OTho5cqT69Omj9u3bq2rVqnrmmWfyvTIAAAAAgP8vz566jIwMPfbYY5o6dapeeOEFzZs3L89G7ezsFBUVZTPNx8fH+v8mTZpo8eLFt1EuAAAAAOBmefbUpaenKyYmRoGBgdq8ebMyMzOLoi4AAAAAQD7kGepGjRolLy8v9erVSxcvXtTYsWOLoi4AAAAAQD7kGupu3GS8Zs2aevXVV+Xk5KTg4GB5eHhIkr7//vuiqRAAAAAAkKtcf1N39epV9ezZU0899ZT8/PxUsWJFXb58Wbt379aGDRv00ksvFWWdAAAAAIAc5BrqXnzxRbVs2VLLly/X4sWLdenSJVWoUEGNGjXSpEmT5OrqWpR1AgAAAABycMurX7q4uKhjx47q2LFjUdUDAAAAACiAPC+UAgAAAAC4d92XoS41ndsuFCe2PwAAAFB08rz5uCRt2rRJp06dUr169eTl5SVnZ+fCruuOODvaq2G/2cVdRom1Y2zX4i4BAAAAKDHyDHWffPKJfvvtNyUmJsrR0VHTpk3TJ598UhS1AQAAAADykOfwyx07dmjMmDFydXVV27Ztdfr06aKoCwAAAACQD3mGuszMTKWmpspisSgzM1N2dvflz/AAAAAAwJTyHH7ZrVs3tWvXThcvXlSHDh3UvXv3IigLAAAAAJAfeYa61q1b64knntDJkyfl4eGh8uXLF0VdAAAAAIB8yDPUrVu3TkuWLFFqaqp12hdffFGoRQEAAAAA8ifPUDd69GhFRUXJ3d29KOoBAAAAABRAnqHuoYceUuPGjYuiFgAAAABAAeUZ6p599lmFhobK29vbOi0mJqZQiwIAAAAA5E+eoS4uLk49e/ZUmTJliqIeAAAAAEAB5BnqKlWqpODg4KKoBQAAAABQQHmGulKlSun111/XI488IovFIknq3bt3oRcGAAAAAMhbnqGuWbNmRVEHAAAAAOA25Brq9u7dq7p166py5cpFWQ8AAAAAoAByDXWbNm1S3bp19d1332Wb99RTTxVqUQAAAACA/Mk11PXq1UuSFBAQoA4dOlinz549u/CrAgAAAADkS66hbsWKFVq3bp22bNmizZs3S5KysrJ0+PBhde3atcgKBAAAAADkLtdQFxQUpMqVK+vSpUsKDQ2VJNnZ2cnDw6PIigMAAAAA3Fquoc7d3V2NGzdW48aNi7IeAAAAAEAB2BV3AQAAAACA20eoAwAAAAATy/Pm4xMnTrR57OjoqKpVqyo4OFiOjo6FVhgAAAAAIG959tQdOnRIJ06cUKVKlXTmzBlt2rRJGzZs0MCBA4uiPgAAAADALeQZ6i5fvqxx48YpLCxMMTExsrOz09ixY3X69OmiqA8AAAAAcAt5hrorV67o4sWLkqSkpCRduXJF6enpunbtWqEXBwAAAAC4tTx/U/fuu++qY8eOcnNz09WrVzV48GDNnDlTr7zySlHUBwAAAAC4hTxDXbNmzfT000/r4sWLqlixoiwWi5o2bVoUtQEAAAAA8pBnqNu4caO++uorpaamWqfNnj27UIsCAAAAAORPnqEuJiZGAwcOVNWqVYuiHgAAAABAAeQZ6qpVq6YnnniiKGoBAAAAABRQnqGuYsWKGjp0qB555BFZLBZJUmhoaKEXBgAAAADIW56hrnr16pKkCxcuFHoxAAAAAICCyTXU/fbbb6patapeeOGFoqwHAAAAAFAAuYa6mTNnasCAARo6dKjNdIvFwtUvAQAAAOAekWuoGzBggCQpLi6uyIoBAAAAABRMnr+pW7p0qaZNm2Zzn7q1a9cWalEAAAAAgPzJM9R98cUXmjJliqpVq1YU9QAAAAAACiDPUOfh4SFPT8+iqAUAAAAAUEB5hrpSpUqpZ8+e8vf3t96nrnfv3rd8TVZWliIjI3Xo0CE5OTlpxIgROQbDIUOGyN3dXX379r3N8gEAAACgZMsz1D399NMFbnTNmjVKS0tTfHy8du3apVGjRmnKlCk2z1mwYIEOHz6sxx57rMDtAwAAAACuyzXU7d27V3Xr1lXlypUL3OiOHTsUFBQkSapfv7727dtnM3/nzp3avXu3QkNDdezYsQK3DwAAAAC4LtdQt2nTJtWtW1ffffddtnlPPfXULRtNTk6Wm5ub9bG9vb0yMjLk4OCgc+fOaeLEiZo4caJWrVqVryJTU1N14MCBfD1Xkvz9/fP9XBSOguwv4AY+u8WPzy4AlCyce4vX3Trv5hrqevXqJUmKiYmxmX7u3Lk8G3Vzc1NKSor1cVZWlhwcri9q9erVSkpKUq9evXT+/Hldu3ZN3t7eateuXa7tOTs784YzGfYXYE58dgEAKDoFOe/eKgDm+Zu6f//735o3b57S09N17do11axZM8feu5sFBARo/fr1Cg4O1q5du+Tr62ud17VrV3Xt2lWStGTJEh07duyWgQ4AAAAAkDu7vJ6QkJCghIQEhYSEaOXKlapSpUqejbZs2VJOTk4KCwtTTEyMBgwYoOXLlys+Pv6uFA0AAAAAuC7Pnrpy5crJyclJKSkp8vT01F9//ZVno3Z2doqKirKZ5uPjk+159NABAAAAwJ3Js6euatWqWrx4sVxcXDR+/HglJycXRV0AAAAAgHzIs6euX79+Sk5OVqtWrfTNN9/o008/LYq6AAAAAAD5kGdP3VtvvaV//OMfcnNzU3h4uGrVqlUUdQEAAAAA8iHPnjp3d3fNmjVLXl5esrO7ngHzuk8dAAAAAKBo5Bnqypcvr4MHD+rgwYPWaYQ6AAAAALg35BrqPvjgA02YMCHbzccBAAAAAPeOXH9Td/HixaKsAwAAAABwG3LtqTt16pQ++eSTHOf17t270AoCAAAAAORfrqGuVKlS8vLyKspaAAAAAAAFlGuoq1Spktq2bVuUtQAAAAAACijX39TVqVOnKOsAAAAAANyGXENd//79i7IOAAAAAMBtyDXUAQAAAADufYQ6AAAAADAxQh0AAAAAmBihDgAAAABMjFAHAAAAACZGqAMAAAAAEyPUAQAAAICJEeoAAAAAwMQIdQAAAABgYoQ6AAAAADAxQh0AAAAAmBihDgAAAABMjFAHAAAAACZGqAMAAAAAEyPUAQAAAICJEeoAAAAAwMQIdQAAAABgYoQ6AAAAADAxQh0AAAAAmBihDgAAAABMjFAHAAAAACZGqAMAAAAAEyPUAQAAAICJEeoAAAAAwMQIdQAAAABgYoQ6AAAAADAxQh0AAAAAmBihDgAAAABMjFAHAAAAACZGqAMAAAAAEyPUAQAAAICJEeoAAAAAwMQIdQAAAABgYoQ6AAAAADAxh8JoNCsrS5GRkTp06JCcnJw0YsQIeXp6WuevWLFCs2bNkr29vXx9fRUZGSk7O/IlAAAAABRUoSSpNWvWKC0tTfHx8erTp49GjRplnXft2jVNmDBBs2fP1oIFC5ScnKz169cXRhkAAAAAcN8rlFC3Y8cOBQUFSZLq16+vffv2Wec5OTlpwYIFcnFxkSRlZGTI2dm5MMoAAAAAgPteoQy/TE5Olpubm/Wxvb29MjIy5ODgIDs7O1WqVEmSFBcXp6tXr+rJJ5+8ZXupqak6cOBAvpfv7+9/e4XjrinI/gJu4LNb/PjsAkDJwrm3eN2t826hhDo3NzelpKRYH2dlZcnBwcHm8dixY3X8+HHFxsbKYrHcsj1nZ2fecCbD/gLMic8uAABFpyDn3VsFwEIZfhkQEKCEhARJ0q5du+Tr62szf+jQoUpNTdXkyZOtwzABAAAAAAVXKD11LVu21MaNGxUWFibDMBQdHa3ly5fr6tWrqlOnjhYvXqzAwEB169ZNktS1a1e1bNmyMEoBAAAAgPtaoYQ6Ozs7RUVF2Uzz8fGx/v/gwYOFsVgAAAAAKHG4ORwAAAAAmBihDgAAAABMjFAHAAAAACZGqAMAAAAAEyPUAQAAAICJEeoAAAAAwMQIdQAAAABgYoQ6AAAAADAxQh0AAAAAmBihDgAAAABMjFAHAAAAACZGqAMAAAAAEyPUAQAAAICJEeoAAAAAwMQIdQAAAABgYoQ6AAAAADAxQh0AAAAAmBihDgAAAABMjFAHAAAAACZGqAMAAAAAEyPUAQAAAICJEeoAAAAAwMQIdQAAAABgYoQ6AAAAADAxQh0AAAAAmBihDgAAAABMjFAHAAAAACZGqAMAAAAAEyPUwXSMjNTiLqHEYtsDyElGemZxl1Cisf0BOBR3AUBBWRyc9UtU3eIuo0SqMXRvcZcA4B7k4GiviX2WF3cZJdY740OKuwQAxYyeOgAAAAAwMUIdAAAAAJgYoQ4AAAAATIxQBwAAAAAmRqgDAAAAABMj1AEAAACAiRHqAAAAAMDECHUAAAAAYGKEOgAAAAAwMUIdAAAAcpWRllbcJZRobH/kh0NxFwAAAIB7l4OTk0Z2eaW4yyixBs1ZXNwlwAToqQMAAAAAEyPUAQAAAICJEeoAAAAAwMQIdQAAAABgYoQ6AAAAADAxQh0AAAAAmFihhLqsrCwNHTpUoaGhCg8P18mTJ23mr1u3Tu3bt1doaKgWLlxYGCUAAAAAQIlQKKFuzZo1SktLU3x8vPr06aNRo0ZZ56WnpysmJkYzZsxQXFyc4uPjdf78+cIoAwAAAADue4US6nbs2KGgoCBJUv369bVv3z7rvMTERNWoUUPu7u5ycnJSw4YNtX379sIoAwAAAADuew6F0WhycrLc3Nysj+3t7ZWRkSEHBwclJyerTJky1nmlS5dWcnLyLdtLTU3VgQMHClTDnB6PFaxo3DUF3Ve3pQPDdotDUezb6S2mF/oykLMi+ezivvVsz1rFXUKJVRSf3XaDhhf6MpCzIjk2t6tW+MtANgXdt6mpqbnOK5RQ5+bmppSUFOvjrKwsOTg45DgvJSXFJuTlpH79+oVRJgAAAACYXqEMvwwICFBCQoIkadeuXfL19bXO8/Hx0cmTJ3Xp0iWlpaVp+/btatCgQWGUAQAAAAD3PYthGMbdbjQrK0uRkZE6fPiwDMNQdHS09u/fr6tXryo0NFTr1q3TpEmTZBiG2rdvr1dfffVulwAAAAAAJUKhhDoAAAAAQNHg5uMAAAAAYGKEOgAAAAAwMUJdAU2bNk3du3dXjx499Prrr2vfvn365Zdf1Lx5c908kjU9PV3NmzfXlStX5Ofnp48//timnREjRqh58+Y5LmPEiBH67bffFBsbq+eff17h4eEKDw9XSEiIpkyZIklasmSJnnnmGYWHh6tz587q3r27zp07J0mKiIhQSEiI9XXh4eE6e/as5s+fr02bNhXSljGXLVu26MMPP7yjNqZNm6Y9e/bkOn/OnDmSpISEBMXHx+erpiZNmlj3Wbt27fTee+8pLS3tjuq83xXWdvvwww+1ZcuWu1LjzZ/XG//Wrl17V9q+2bZt23Tw4MG73u797u/Hg9WrV6tNmzbq2rWr3nnnHZvnPvnkk5Ku79PmzZvb3JLnbr5nSrqiPtfOnz/fZt758+cVGRl519dLUok8F2/ZskV+fn5auXKlzfSQkBBFRETk+JolS5Zo3LhxkqT4+Hilp6frwIEDmjhx4l2vL6/z9I33yOnTp1W7dm2b+y/Pnz9fsbGxkqTmzZvr1VdfVZcuXdSuXTvNnTv3rtd6r7r5XNylSxeFhYVl29/5MXLkSJ09ezbHefn9PpWb6dOnKzw8XC+99JLN94bMzMzbbvNeUii3NLhfHT16VOvWrdP8+fNlsVh04MAB9e/fX8uWLVONGjW0detWNW7cWJK0bt06NW7cWGXKlFG5cuW0bds26736MjMzbQ4IN9u1a5ccHBxUtWpVSVL37t3VqVMnSVJaWpqCg4PVsWNHSVKbNm3Ut29fSdcPeFOnTtXQoUMlSf369VPTpk1t2u7QoYNee+01NWrUSPb29nd/A5UwvXr1uuX8KVOmqEuXLtn2w608/vjj+vTTT62P+/Tpo3Xr1qlVq1a3XWdJYIbtdvPntbB8/fXXCg4O1sMPP1yoy7mffffdd/ryyy/11Vdfady4cfrxxx+1dOlSvfzyy9me+9dffyk6OlrR0dFFX+h9rDjOtX9XuXLlQgt1JfVc7O3trRUrVig4OFiSdOjQIf3111/5eu3nn3+ul19+Wf7+/vL397/rtRXkPO3m5qYBAwbo66+/lpOTU7b5M2bMkLOzs/U7W6tWrVSxYsW7We496+ZzcUpKisLDw+Xl5VWgfTZo0KBc5xVkP+WkZ8+e6tmzp7Zs2aIFCxbYfG+4HxDqCqBChQo6e/asFi9erKZNm8rf31+LFy+WJHXs2FFLly61nmi+/vprvf3225IkBwcHNWrUSBs3btTTTz+tDRs2qEmTJvr222+zLSMuLk6vvfZajstPSkpSRkaGnJ2ds837888/9Y9//OOW9Ts4OKh27dr64Ycf9OyzzxZo3UuKjRs3asKECXJ2dla5cuUUHR2tMmXKaNiwYdq3b58qVaqkM2fOaMqUKZo4caKCg4Pl4eGhAQMGyMHBQfb29hozZoyWLFmiP//8U5GRkapXr56OHTumvn37avLkyVqzZo0yMzPVqVMnhYWF5VpLWlqazp07J3d3d0nS+PHjtW3bNhmGoe7du6t169bas2ePhg0bptKlS6tixYpydnbWO++8o3/+858qV66cmjZtqjfeeKOoNt894ebtlpmZqaFDh+q3335TUlKSmjZtqg8++EARERFycnLSmTNndO7cOY0aNUq1a9fW3LlztWjRIlWuXFl//PGHpOs9AQMHDtSpU6eUmZmp1157TcHBwQoPD5efn5+OHDkiV1dXBQYGasOGDbp8+bJmzJhh3W+3cvnyZfXr10/JycnKzMzU+++/ryZNmqhNmzaqWbOmnJycNGzYMA0aNEhJSUmSpMGDB8vPz08RERH65ZdflJqaqtdff101atTQ//3f/+nnn39WrVq19OCDDxbqdr4fLV26VHPmzNHMmTOt+69Pnz6KjY3V448/ni0AvPzyy9q5c6fWr1+vZs2aFUfJ96XiPtdK0unTp9W7d28tXLhQISEhatSokQ4dOiSLxaLJkyerTJkyOR6Tt27dau1JunbtmkaPHi1HR8dsx+SSeC5++OGHdeLECV2+fFlly5bVsmXLFBISol9//VVPPvmkNm7cKOl6j/fN58ZFixbp/Pnz+vDDD9WtWzfrl/HnnntOAQEBOn78uCpWrKjY2FhlZWXd1vF67dq11vP0+PHjtW/fPqWkpMjHx0cxMTE26+Hp6anAwEB9+umn6t+/f67re+3aNTk7O+d5L+b7VenSpRUaGqrVq1fL398/x8/L7t27NXLkSBmGoSpVqmjcuHF64403FBkZqUuXLmn06NFycHBQ2bJlNW7cOP33v/+17qcZM2bou+++k4ODgwIDA9WvXz/Fxsbq9OnT+uOPP3T27FkNGDBAQUFBedYaGxurnTt36urVqxo5cqR++uknrVixQhaLRcHBweratat+/fVXDRkyRKmpqXJ2dtbw4cNVrdq9dcN2hl8WQIUKFTRlyhT973//U2hoqFq1aqX169dLklq0aKFt27bp2rVrOnfunC5cuGBz0/Q2bdpYu6FXrFihkJCQHJexdetWm/v6ffXVV+rSpYueffZZffjhhxoxYoTc3Nys7dwYbvbll1/a/AVj7Nix1m7lG0M2JcnPz09bt269a9vkfmIYhoYMGaKJEydqzpw5euyxxzRlyhStXbtWly5d0uLFixUdHa1ff/3V5nU//fSTateurZkzZ+qtt97Sn3/+qX/+859yd3e3+Uvv/v37lZCQoEWLFmnBggU6evSo/n7x2c2bNys8PFzBwcFq166dWrZsqSZNmujHH3/U6dOntWDBAs2ePVtTp07V5cuX9fHHH2vUqFGaPXu2atSoYW3n/Pnz+vLLL0tMoMttu/3666+qX7++vvzyS82fP99miNWDDz6oL7/8UuHh4YqPj9eVK1c0e/ZsLVy4UJMnT1Z6erqk673g5cuX14IFCzRz5kxNmDBBFy9elCTVq1dPs2bNUlpamkqVKqWZM2eqVq1a2rZtW7Yab3xew8PD9d5770m63pv7xBNPaO7cufrss880aNAgZWVl6erVq3r77bf1ySefaOrUqXr88ccVFxen4cOHKzIyUsnJydqyZYsmTpyoL774QpmZmapTp46CgoLUr18/At1t2L59uxYuXKg///zTZijOAw88oPfffz/Hvx7b29tr1KhRio6OtoZu3LniONfeSkpKil544QXNmTNHDzzwgBISEnI9Jh85ckRjx47V7Nmz1bx5c61evVpS9mNyST0Xt2zZUt9//70Mw9CePXvydZ/iDh06qHLlytl6VU6dOqX3339f8fHxunjxovbu3XvHx+vk5GSVLVtWM2fO1IIFC7Rr1y79/vvv2Wr64IMPtHHjRm3fvj3bvB49eqhLly5q1aqVGjZsKEdHx4JupvtGxYoVlZSUlOvnZciQIYqJidGiRYvUpEkTJSYmWl+7Zs0atWzZUnPmzNErr7yiy5cvW+cdOnRIq1at0oIFC7RgwQKdPHnSeoxwcnLS9OnTNWjQIH311Vf5rtXb21sLFiyQYRhauXKl5s2bp3nz5mnNmjU6duyYRo8erfDwcMXFxen111+3Dg2+l9BTVwAnT56Um5ub9a82e/fuVa9evdS4cWOVK1dOLVq00Jo1a3T27Fm1b9/e5rUNGzbUsGHDlJSUpEuXLuXaq5aVlWXTnX9j+OW+ffvUu3dv1axZ0zrv5uFcmzZt0ttvv63vv/9eUs7DL6XrQ0o2b958R9vhfpWUlCQ3NzdVqVJFkvTYY4/pk08+Ufny5a1fGipUqCBvb2+b173yyiv64osv1LNnT5UpUybX3+odP35c9erVk729vVxcXDR48OBsz7kxdCEpKUk9evRQ9erVJUmHDx/Wzz//rPDwcElSRkaGzp49q3Pnzumhhx6SdP09duPLTPXq1XMcFnK/ym27lStXTnv37tXmzZvl5uZm8zu7G8NBqlatqv/97386duyYatWqZd1u9erVkyQlJibqiSeekHR92I2Pj49OnTolSapdu7YkqWzZsqpVq5b1/6mpqdlqzGn4ZWJiovVLZ5UqVeTm5mb9AuLl5SXp+r7fvHmzVq1aJel6756bm5uGDBmiIUOGKDk5WS+++OIdbT9cPzbOnDlTixYtUr9+/fTFF19Y57344otas2aN5s2bl+11NWvWVNeuXTVs2DBZLJaiLPm+VRzn2rw88sgjkqRq1aopNTVVZ8+ezfGYXKVKFY0cOVKurq76/fffFRAQICn7MbmknotDQkIUGRkpDw8PBQYG5vic/N5pq3z58taekhv75U6P187Ozrp48aJ69+4tV1dXXb161foHvps5OTkpJiZGffr0sf4k5oabh1/26tVLy5Yt00svvZSvdbrfnD17VlWrVs31O8wff/whHx8fScp2z+q33npLU6dOVbdu3VSlShXrOVmSjh07pkcffdQamAMDA3XkyBFJtuf2gvy2/uZz7tmzZ9W9e3dJ10fC/fLLLzp8+LA+//xzTZ8+XYZh3JNhnZ66Ajh06JAiIyOtBwAvLy+VKVPGOia+Q4cOWrFihdasWZPtS5bFYtHTTz+tyMhItWjRItdlODs75/iDzTp16uiNN95Q7969lZWVlW1+tWrVcjzw/N3ly5dVoUKFPJ9XEpUvX17JycnWC85s3bpVNWvW1EMPPaRdu3ZJuv7hPnHihM3r1q5dq4YNG2rWrFlq1aqVpk+fLin7icnb21v79+9XVlaW0tPT9dprr+V6wClfvrzGjh2rwYMH69y5c/L29lbjxo0VFxenWbNmqXXr1qpevbqqVq2qo0ePSpJ2795tfb2dXcn8aP99uy1ZssQ6TKpHjx66du2adb/8/Qu4h4eHjh49qmvXrikzM1MHDhyQJPn4+Fj/GpucnKzDhw9bQ+Odurnt33//XZcvX1a5cuUk/f996O3tre7duysuLk4TJkxQSEiIzp07p59//lmTJk3StGnTNHbsWGVkZMhiseT7CxFseXp6ytnZWV26dJGjo6PNCAdJioyM1IwZM5SSkpLttV26dNGlS5dK5Jf0wlCc59rc/P14kdsxefDgwYqOjtaoUaP0wAMPWD+Pfz8ml9RzsYeHh65evaq4uDibfZeRkaGUlBSlpaVZz2k3s1gs2b775PRHlDs9XickJOjXX3/VJ598ot69e9ucM/6udu3aatOmjc0fgG7m5OSkihUr5uu72f0oOTlZixYtUqtWrXL9vDzwwAPW71TTpk2zdkxI0vLly9W2bVvFxcXpoYce0sKFC63zvL29tWfPHmVkZMgwDG3bts0aym73j2s3n3Nr1aql2bNnKy4uTu3atZOvr6+8vb3Vt29fxcXFadiwYXr++edvc8sUHnrqCuC5555TYmKiOnToIFdXVxmGoY8++sg6XtrHx0dXr16Vj49PjmOoQ0JC1L59e0VFReW6jICAAP388882f5G4oUOHDlq1apXmz58vFxcXrVixQrt375a9vb1SUlI0bNiwPNdh9+7d1qu3lXQbN25Uu3btrI/Hjx+vESNG6N1335XFYpG7u7tiYmJUvnx5JSQkKCwsTJUqVVKpUqVs/kJTp04d61huOzs7DRgwQNL190Pfvn2tfzX09/dXUFCQOnXqpKysLHXq1OmWfymuVauWwsPDNWLECH322WfaunWrOnfurKtXr6pFixZyc3PTxx9/rIEDB8rV1VWOjo7WXsab9ejRQ1OnTi0xPXc3b7d3331XvXv31o4dO+Ti4iJPT09raP+7ChUq6P3331dYWJgqVKggFxcXSdd/wzNkyBB16tRJqampeuedd+7aj97ffPNNDRw4UP/5z3907do1RUVFycHB9rD81ltvadCgQVq4cKGSk5P1zjvvqHLlyjp//rxefvllubq6qkePHnJwcNCjjz6qcePGqXr16ta/fqLgoqOj9fLLL8ve3t56UYcKFSooIiJC//rXv7I932KxKDo6OtehfiiY4jjXTps2TYsWLZJ0/bdAf/8d1d81b948x2PySy+9pI4dO6ps2bKqVKlSrsebknwuDg4O1rfffisvLy9rL1rXrl0VGhqq6tWr5zh8PDAwUL169crx83ezOz1e16tXT5MnT1bHjh3l5OQkDw+PXPehdP34fGPY3w09evSQnZ2dsrKyVLVq1RI1kuLGTyHs7OyUmZmpd999V97e3vLy8srx8zJs2DANHDhQdnZ2qly5srp3767Zs2dLkurWrauIiAjr95uoqCjrUFk/Pz+1bt3a+n2qYcOGatGixV25+vPDDz+sJk2aqFOnTkpLS1O9evVUpUoV9e/f3/rHpmvXrt3ygi7FxWLwZ917ys6dO/Xdd9/lODTvTmVkZOi1117TV199VaKuuHWnEhMTdfDgQb3wwgtKSkpSmzZttH79+nsiJM2dO1etW7dWhQoV9Omnn8rR0THbJdgBALYK81ybF87FAApDyRyjdQ9r0KCBMjMz9dtvv931tuPj4/Xmm29yEimgatWqacWKFerYsaN69uypvn373hOBTrr+I+QePXqoc+fOOnjwYLYx6QCA7ArzXJsXzsUACgM9dQAAAABgYvTUAQAAAICJEeoAAAAAwMQIdQAAAABgYoQ6AECJs2XLFvn5+WnlypU200NCQhQREZHn61NTU9W8efNbtv/hhx/ecZ0AAOQHoQ4AUCJ5e3trxYoV1seHDh3SX3/9VYwVAQBwe7j5OACgRHr44Yd14sQJXb58WWXLltWyZcsUEhKiX3/9VcuWLdOsWbPk5OSkmjVrKioqSmlpaerbt68uX76sGjVqWNs5dOiQRowYIUkqV66coqOji2uVAAAlFD11AIASq2XLlvr+++9lGIb27NmjBg0a6NKlS4qNjdWsWbM0f/58lSlTRvHx8frmm2/k6+uruXPnKiwszNrGkCFD9PHHHysuLk5NmzbV9OnTi3GNAAAlET11AIASKyQkRJGRkfLw8FBgYKAkKSsrS7Vq1ZKbm5sk6bHHHtOGDRskSUFBQZKkRx99VA4O10+hiYmJGjZsmCQpPT1dXl5eRb0aAIASjlAHACixPDw8dPXqVcXFxal37946deqULBaLEhMTdfXqVbm6umrr1q3y8vKSxWLRrl271KJFC+3fv18ZGRmSJC8vL40ePVoPPvigduzYofPnzxfzWgEAShpCHQCgRAsODta3334rLy8vnTp1SuXLl1ebNm3UtWtX2dnZqUaNGurbt6/s7e01YMAAderUSd7e3nJ0dJQkRUZGqn///srMzJQkjRw5UufOnSvOVQIAlDAWwzCM4i4CAAAAAHB7uFAKAAAAAJgYoQ4AAAAATIxQBwAAAAAmRqgDAAAAABMj1AEAAACAiRHqAAAAAMDECHUAAAAAYGKEOgAAAAAwsf8HqkCRfP0IHFIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1080x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize = (15,5))\n",
    "sns.barplot(x = 'Model', y = 'Training time (sec)', data = df_accuracy)\n",
    "plt.title('Training time for each model in sec', fontsize = 15)\n",
    "plt.ylim(0,1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f720922d",
   "metadata": {},
   "source": [
    "## **Logistic Regression**<br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "3e3ff60c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 85.8 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "lr = LogisticRegression(solver='liblinear', penalty ='l2' , C = 1.0)\n",
    "lr.fit(dtv, y_train)\n",
    "pred = lr.predict(test_dtv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "e39f4a28",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  96.93877551020408\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy: ', accuracy_score(y_test, pred) * 100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67ba7708",
   "metadata": {},
   "source": [
    "**Classification Report**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "ca29c054",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       184\n",
      "           1       0.75      0.75      0.75        12\n",
      "\n",
      "    accuracy                           0.97       196\n",
      "   macro avg       0.87      0.87      0.87       196\n",
      "weighted avg       0.97      0.97      0.97       196\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96104809",
   "metadata": {},
   "source": [
    "**Confusion Matrix**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "18cde83d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAFxCAYAAAB9Z2NTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAZtUlEQVR4nO3de1RVdf7/8dfhcgQv3LJQl5dAcaQpHRpNmRjL1MZs/Gkt03DAqMZsdHKhpqLpRJqJo6T5nUFHE29JNqCoqWPLxAmtQR1rxrzk/W4C3hVBxLN/f7jiO34Lgelsjn58PtZqLeBw9ucNS59+2uecfRyWZVkCABjLy9MDAADsRegBwHCEHgAMR+gBwHCEHgAMR+gBwHA+nh7g/1rZL9LTIwA/qGjEOk+PAFQoNqpxhbexowcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoTdEUPPW+sXYBZKkgGatFPPWEj365gf62StvSw5H+fc56wXriXfXysvX6alRcZdzua5r+awpmvuHIUpPTtTZUyc9PZLxCL0BWvz6Zf1swAR5+daSJP3k2cHam52mz9+Kk5evU6FRj0mS7m39qKJHv69aAfd4clzc5fZs+4ck6eXxM9TpuQR9smimhycyH6E3QFH+UW2dPqT88wuHd8tZJ1CS5ONXR66yshs3uCx98c7LulZ0wRNjApKkyHYx6jFgmCTpwul81QkM9vBE5vOx46DTpk1TVlaWHP9xymDTpk12LAVJ325dJ//6jco/Lzp1WA+9OE4Rz7yqsiuXdGb3FklS4Y4vPDUicBNvb29lp6Vo99bP1SfxTU+PYzxbQv/ZZ59pw4YNcjo5D+wJD/Yfo8/fitelE/t1f9d++ulvRunr+RM8PRZwk2cGJanL+bN6f+xgDZ6aLqefv6dHMpYtp24iIyN19epVOw6NKigtuqBrxZclSSXnCuRbJ8DDEwH/69+567RxeYYkyddZSw6Hlxxe3h6eymy27OgjIiIUExOj+vXry7IsORwOrV+/3o6l8AP+PWec2r6WKpfruqyya/rXnHGeHgkoF/lIjJbPmqL05ES5rpep2wuD5Mv//dvKYVmW5e6D9u7dW7NmzVJAwP/uJKt6Gmdlv0h3jwO4RdGIdZ4eAahQbFTjCm+zZUffqFEj+fv7c44eAG4DtoT+1KlT6tq1q5o0aSJJcjgcWrJkiR1LAQAqYdvTKwEAtwdbQl9WVqa1a9fq2rVrkqSCggKNHz/ejqUAAJWw5emVo0aNkiR9+eWXOn78uM6fP2/HMgCAKrAl9H5+fho4cKBCQ0OVkpKi06dP27EMAKAKbAm9ZVkqLCxUUVGRrly5ogsXuLYKAHiKLaH//e9/r3Xr1qlnz57q3LmzOnbsaMcyAIAqsOXB2Hbt2ql58+Y6duyY/va3vykoKMiOZQAAVWBL6BcvXqwFCxYoIiJC+/fv16BBg9SzZ087lgIAVMKW0GdmZurjjz9WrVq1VFxcrLi4OEIPAB5iyzn6e+65R97eN65G5+fnx6kbAPAgW3b0lmWpV69eioqK0q5du1RWVqbhw4dLklJTU+1YEgBQAVtC/+qrr5Z/3KNHDzuWAABUkdtD/+mnn6pLly66dOmS0tLS5HQ6NXDgQNWuXdvdSwEAqsCt5+inTp2qFStW6Pr165owYYKuXLmi4OBgJScnu3MZAEA1uHVHv3PnTs2bN09lZWX6+9//rs8++0z+/v6KjY115zIAgGpw647+u2fabN++XS1btpS//403+/3uKpYAgJrn1h29t7e3Nm3apOzsbD355JOSpC+++OKmtxQEANQst+7o33jjDWVlZalBgwaKjY3Vxo0blZKSorFjx7pzGQBANdjy5uA/Bm8OjtsVbw6O29mt3hzcllfGAgBuH4QeAAxnS+gzMzNv+nzhwoV2LAMAqAK3Putm1apVysnJ0ebNm5WXlydJun79uvbt26f+/fu7cykAQBW5NfS//OUvde+99+r8+fPq27evJMnLy0tNmjRx5zIAgGpw66mbwMBAtW/fXunp6SouLtb27dt1/vx5hYaGunMZAEA12HKOPjU1VVlZWfLx8dHy5cuVkpJixzIAgCqw5TLFW7du1ZIlSyRJL7zwgvr06WPHMgCAKrBlR19WViaXyyXpxpuQOBwOO5YBAFSBLTv67t27KzY2Vm3atNH27dvVvXt3O5YBAFSBLaF/6aWXFBMTo4MHD6p3795q2bKlHcsAAKrAraFfvnz59762a9cu7dq1S7169XLnUgCAKnJr6A8cOHDT55ZladmyZfLz8yP0AOAhbg398OHDyz8+cuSIkpKS9Pjjj2vMmDHuXAYAUA22nKNfvHixFixYoNGjR6tTp052LAEAqCK3hj4/P1+jR49WYGCgMjMzFRgY6M7DAwD+C24N/a9//Wv5+vqqQ4cOGj9+/E23paamunMpAEAVuTX0f/7zn915OACAG7g19I888og7DwcAcAPeYQoADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADOdT0Q2bNm2q8E4xMTG2DAMAcL8KQ7969eoK70ToAeDOUWHoJ02a9INfLygosG0YAID7VRj678yYMUMZGRm6du2aSkpKdP/9999ytw8AuL1U+mBsbm6ucnNz1aNHD61Zs0ahoaE1MRcAwE0qDX1QUJCcTqeKiorUrFkzFRcX18RcAAA3qTT0DRo0UFZWlvz9/ZWamqrLly/XxFwAADdxWJZl3eobXC6Xvv32WwUGBio7O1vR0dFq0aKFbQOt7Bdp27GBH6NoxDpPjwBUKDaqcYW3Vfpg7MqVK8s/rlevnnbs2GFr6AEA7lVp6A8cOCBJsixLu3fvVlBQkHr16mX3XAAAN6k09MOHDy//2LIsDRw40NaBAADuVWnoS0tLyz8uLCzU8ePHbR0IAOBelYa+W7ducjgcsixLfn5++u1vf2vrQDzgBQDuVWnop0+frtatW5d/vmXLFlsHAgC4V4Wh/+c//6n9+/dr/vz5evHFFyXdeKrl4sWLtWrVqhobEADw41QY+oCAAJ0+fVqlpaUqLCyUJDkcDo0YMaLGhgMA/HiVvmAqPz9fZ8+eVWRkpD799FM99thj8vX1tW2gD7/iwV4AqK5bvWCq0ksgTJw4Uf/6178kSYcOHVJSUpLbBgMA2K/S0Ofn5ys2NlaSNGDAAK5HDwB3mCq9Z+yhQ4ckSUeOHJHL5bJ1IACAe1X69MoxY8YoMTFRZ86ckZ+fn5555pmamAsA4CaV7ujbtGmjCRMm6Be/+IWKi4t15syZmpgLAOAmFe7oS0tLtXr1ai1evFhOp1OXL1/W+vXr5efnV5PzAQB+pAp39E888YT27NmjqVOnKiMjQ/fddx+RB4A7UIU7+v79+2vVqlU6ceKEevfurUqebg8AuE1V+oKpLVu2KDMzU7m5uerdu7d69uypli1b2jYQL5gCgOq71QumKg39dy5evKgVK1Zo6dKlWr58ubtm+x5CDwDV55bQ1xRCDwDV96MugQAAuLMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwnI+nB4A9XK7rWjn7XZ05eUwOLy/1enWkQho08vRYgMqulWr5zCk6V/CtavnX1tMvDdE9DRt7eiyjsaM31J5t/5AkvTx+hjo9l6BPFs308ETADdvWr5HTz18D3v6Tur/4mtbM+x9Pj2Q8dvSGimwXo5YPR0uSLpzOV53AYA9PBNxQeOKIIn72iCSpfqMmKjxx1MMTmc+20B87dkwbNmzQ1atXy782YMAAu5bDD/D29lZ2Wop2b/1cfRLf9PQ4gCSpQbPm2vvlP9Sq3aM6vn+3Lp09LZfrury8vD09mrFsO3UzaNAgXbhwQU6ns/w/1LxnBiXptWkL9PGcVJWWFHt6HEBRnZ5SLf86mj9+uPZu+4cahkcQeZvZtqNv2LChXnvtNbsOj0r8O3edLp4t1C979ZOvs5YcDi85+MuE28DJA9+oaasH1e2FQTpxYI/O5p/09EjGc1iWZdlx4A8//FAnTpxQixYtyr/Wq1evyu/31XE7xrnrlJYUa/msKbp8/qxc18sU0zNWrdo+6umxABVdvKCsGW/r2tVi+dWuq/838HUFhNT39Fh3vNioip+5ZFvo4+PjFR4eroCAgBsLORwaNmxYpfcj9ABQfbcKvW2nbpxOp9566y27Dg8AqCLbQt+oUSP95S9/0QMPPCCHwyFJiomJsWs5AEAFbAt9WVmZDh8+rMOHD5d/jdADQM2zLfSTJk266fOCggK7lgIA3IJtoZ8xY4YyMjJ07do1lZSU6P7779fq1avtWg4AUAHbXjCVm5ur3Nxc9ejRQ2vWrFFoaKhdSwEAbsG20AcFBcnpdKqoqEjNmjVTcTGvygQAT7At9A0aNFBWVpb8/f01depUXb582a6lAAC3YNsLplwul06dOqWAgABlZ2crOjr6plfJVoQXTAFA9XnkBVPnzp1Tenq6Dh8+rIiICN177712LQUAuAXbTt0kJiYqPDxcr7/+uho3bqyRI0fatRQA4BZsfeORfv36SZJatWqltWvX2rkUAKACtu3ow8PDtWLFCuXn5ysnJ0dBQUE6dOiQDh06ZNeSAIAfYOvVKyXp4sWL8vb2Vp06dW4s6HBo4cKFFd6PB2MBoPpu9WCs23f0O3fuVK9evTR37lzFx8ersLBQRUVFSkhI0KJFi24ZeQCA+7k99NOmTVNKSoqcTqemT5+uOXPmaOnSpZozZ467lwIAVIHbH4y1LEutWrVSfn6+iouL9dOf/lSSyi9VDACoWW7f0btcLknSxo0bFR0dLUkqLS3VlStX3L0UAKAK3L6jj46O1vPPP69Tp05p5syZOnr0qJKTk9W9e3d3LwUAqAJbnnVz4MABhYSEKDg4WEePHtWePXvUtWvXKt2XZ90AQPXV+CUQmjdvXv5x06ZN1bRpUzuWAQBUgW0vmAIA3B4IPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEIPQAYjtADgOEclmVZnh4CAGAfdvQAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCG8/H0ALhh8+bNGjx4sD7++GM1bNhQkjR16lSFh4fr2Wef/cH7nD9/Xhs3blSPHj1u+vqRI0c0ceJEXb9+XWVlZXrwwQc1fPhweXnx7zrsNXv2bH3xxRfy8vKSw+HQ0KFD9eCDD3p6rLsef/NvI76+vho9erSq+tKGPXv2KCcn53tff/fddxUXF6e5c+dq/vz5Onz4sNavX+/ucYGb7N+/Xzk5OZo3b57S09P1+uuva8yYMZ4eC2JHf1vp0KGDXC6XFi9erLi4uJtuS09P1+rVq+Xj46O2bdtqxIgRmjVrlr755ht99NFH6tu3b/n3NmrUSNnZ2apTp45at26t6dOny8fHR5s3b9asWbPk5eWlwsJC9e3bV7/5zW+0ZcsW/elPf5IklZSUaPLkyfL19dXQoUPVsGFDHT9+XE8//bT27dunXbt26fHHH9ewYcNq9HeD219ISIhOnjyprKwsdezYUZGRkcrKylJ8fLzCwsJ06NAhWZaladOmKSQkRH/4wx906tQpnTt3Th07dlRiYqKSkpLk4+OjkydPqrS0VN27d9eGDRv07bffKi0tTU2bNvX0j3lnsnBbyMvLsxITE62zZ89anTt3tg4dOmRNmTLFWrp0qfXNN99YvXv3tkpLSy2Xy2UNHjzYysnJKb/P/3X16lVr3rx5Vr9+/ay2bdtaw4cPty5cuGDl5eVZTz31lHX16lWruLjY6tKli3X69Gnrgw8+sE6dOmVZlmXNnDnTSktLs44dO2a1b9/eunjxolVQUGA99NBD1rlz56ySkhIrOjq6pn89uEPs2LHDSkpKsh577DHrV7/6lbV27VorLi7Oys7OtizLsj744ANrwoQJ1rFjx6y//vWvlmVZVklJifXII49YlmVZo0aNstLS0izLsqxx48ZZkydPtizLst577z1r3rx5Nf7zmIId/W0mODhYY8aMUVJSkh5++GFJ0sGDB9WmTRv5+vpKktq2bat9+/apTZs2P3iMvLw8JSQkKCEhQUVFRZo8ebLS0tLUqVMnRUVFyel0SpIiIiJ09OhRhYaGauLEiapdu7by8/PL123SpInq1asnp9Op+vXrKygoSJLkcDhs/i3gTnTkyBHVrVtXkyZNkiR9/fXXeuWVV1S/fn116NBBkvTwww8rJydHQUFB+vrrr5WXl6e6deuqtLS0/DgPPPCAJCkgIEDh4eHlH//n96B6OEd/G3riiScUFham7OxsSVJ4eLi2b9+usrIyWZalrVu3KiwsTF5eXnK5XN+7/5QpU/T5559LkurUqaOwsLDyuO/evVvXr19XcXGx9u/fr2bNmmns2LF65513lJKSovvuu6/8MQKCjurYs2ePkpOTdfXqVUlSWFiY6tWrJ29vb+3YsUOS9OWXX6pFixZatmyZ6tWrp9TUVL300ksqKSnhz52N2NHfpt544w3l5eVJkn7yk5/oqaeeUmxsrFwul37+85+rS5cuKigo0N69ezV//nwlJCSU33f69Ol6++23lZqaKqfTqcaNGys5OVk7d+5UWVmZBgwYoPPnz+t3v/udQkJC1LNnT/Xp00cBAQGqX7++CgoKPPRT40725JNP6sCBA3ruuedUu3ZtWZalkSNHasGCBcrOztb8+fPl7++vP/7xjzp9+rSGDRumbdu2yd/fX82aNePPnY24euVdZPPmzVqyZImmTZvm6VFwF4mPj1dycrKaN2/u6VHuWpy6AQDDsaMHAMOxowcAwxF6ADAcoQcAwxF6GGvz5s2Kjo5WfHy84uPj1adPHy1atKjax5k6daqWLVum3bt3l18q4oesW7dO+fn5VTpmbm6ukpKSqj0L8N/gefQwWocOHcqfTlpaWqpu3bqpZ8+eCggIqPaxIiMjFRkZWeHtCxcuVHJyskJDQ//reQE7EHrcNS5fviwvLy8lJCSocePGunjxombPnq3k5GQdOXJELpdLiYmJat++vT755BPNnDlTISEhunbtmsLDw296HUJmZqY+/PBDuVwude7cWQ899JB2796tUaNGKSMjQx999JFWrVolh8Oh7t27q3///jpw4IDGjBkjf39/+fv7KzAw0NO/EtwlCD2MlpeXp/j4eDkcDvn6+mrcuHF6//331aNHD3Xt2lUZGRkKDg7WO++8o3PnzikuLk6rV6/WlClTlJmZqaCgIL3yyis3HfPMmTOaM2eOVq5cKafTqZSUFLVr106RkZFKTk7W0aNHtWbNGmVkZMjhcCghIUExMTF67733NGTIED366KOaPXu2Dh486KHfCu42hB5G+89TN995//33FRYWJknau3evtm3bpu3bt0uSysrKdPr0adWtW1fBwcGSpKioqJvuf+zYMUVERMjPz0+SvnfN9b179+rkyZPll6W4cOGCjh49qn379ql169aSblzci9CjpvBgLO5K3104Kzw8XE8//bQWLVqkOXPmqFu3bgoICNClS5d09uxZSTeuwvifmjZtqoMHD5ZfTXHIkCHKz8+Xw+GQZVkKDw9XixYttHDhQi1atEjPPvusWrZsqfDwcH311VeSVH6RL6AmsKPHXe3555/X2LFjFRcXp8uXL6tfv35yOp2aNGmSXn75ZQUGBsrH5+a/JiEhIRowYIDi4uLkcDjUqVMnhYaGKioqSiNHjlR6erqio6MVGxur0tJStW7dWqGhoXrzzTc1dOhQzZ07VyEhIapVq5aHfmrcbbgEAgAYjlM3AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8Ahvv/W2MWLxfXv2oAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "confusion_matrix = pd.crosstab(y_test, pred, rownames=['Actual'], colnames=['Predicted'])\n",
    "plt.figure(figsize = (6, 6))\n",
    "sns.heatmap(confusion_matrix, annot = True, cmap = 'Paired', cbar = False, fmt=\"d\", xticklabels=['Not Spam', 'Spam'], yticklabels=['Not Spam', 'Spam']);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52dc6b69",
   "metadata": {},
   "source": [
    "## **Support Vector Machine (RBF)**<br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "8b562257",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 811 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "svc = SVC()\n",
    "svc.fit(dtv, y_train)\n",
    "pred = svc.predict(test_dtv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "61b0f647",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  97.44897959183673\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy: ', accuracy_score(y_test, pred) * 100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "feb26d45",
   "metadata": {},
   "source": [
    "**Classification Report**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "a9d9945f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.99       184\n",
      "           1       1.00      0.58      0.74        12\n",
      "\n",
      "    accuracy                           0.97       196\n",
      "   macro avg       0.99      0.79      0.86       196\n",
      "weighted avg       0.98      0.97      0.97       196\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "54afd9e3",
   "metadata": {},
   "source": [
    "**Confusion Matrix**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "238940e3",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAFxCAYAAAB9Z2NTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAZvklEQVR4nO3de1TUdf7H8dcgjOCFWybqz0vgJSnTH62ltG5pamuWP91+roYLLl3MtrZWIxVNN8o1cYVVO7vqat6TbEVRU9f9mbih9UNdq5/XvCACpoB4DeQizvf3hyfOuongNl/GPj4f53gOM8PM5w1Hn37Od2a+47AsyxIAwFhenh4AAGAvQg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhvP29AD/at2wcE+PAFxXyZjNnh4BqFZURMtqb2NHDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhvTw8A9whs21n3RMXps9/9Uv5tOqrzswmyXJUqOXVcX86fJFnW1W90ONRtzFzl705XzpYPPTozbk8ul0sbFs5SQU6W6nk79V8j43RHs//w9FhGY0dvgHZPPqf/HDFZXj71JUl3P/WyDqfN1qdvRcvLx6mQiEeqvrfjkN/I2SjAU6MC+uofn6qyokLPT/6j+gx7Xv+zbK6nRzIeoTdASUGuds18teryheMH5Wx4Nebevg3lqqyUJDV/8DHJZanw/7Z5ZE5AknK/2qt2//mAJKlV+3t08tghD09kPlsO3cyYMUOpqalyOBxV123fvt2OpSDp1K7N8mvSoupySf5x3ffMJLX/2YuqvPSNzhzcqcYt26vlQ09q16zf6O6nXvLgtLjdlZdekq9fw6rLDq96unLliurVq+fBqcxmS+g/+eQTbd26VU6n046HRw06DZ+gT9+K0TdfH9VdfYfp3l+M05XyUvkGN9VDbyxWgzv/Q67Ky7p0+mud3sN/wKhb9f0aqLystOqyZbmIvM1sCX14eLjKy8sJvYdUlFzQ5dJiSVLZuUIFd4jQ3sWTq26/+79fVtn5IiIPj2h9dycd2v2/6hTZU3lHDiikVainRzKeLaFv3769evTooSZNmsiyLDkcDm3ZssWOpXAd/zd/krq+kiyX64qsystXX3UD3CI6PtBDWXt3671Jr0iyNPDFsZ4eyXgOy/r2dXfuM3jwYM2dO1f+/v5V19V2d79uWLi7xwHcomTMZk+PAFQrKqJltbfZsqNv0aKF/Pz8OHQDALcAW0Kfn5+vvn37qlWrVpIkh8OhFStW2LEUAKAGtr28EgBwa7Al9JWVldq0aZMuX74sSSosLNTbb79tx1IAgBrY8s7YcePGSZI+//xznThxQufPn7djGQBALdgSel9fX40cOVIhISFKTExUUVGRHcsAAGrBltBblqXTp0+rpKREly5d0oULF+xYBgBQC7aE/te//rU2b96sgQMHqnfv3nr44YftWAYAUAu2PBn7wAMPqG3btsrLy9Nf//pXBQYG2rEMAKAWbAn98uXLtWTJErVv315Hjx7VSy+9pIEDB9qxFACgBraEfuXKlfroo49Uv359lZaWKjo6mtADgIfYcoz+jjvuqDrtqK+vL4duAMCDbNnRW5alQYMGKSIiQgcOHFBlZaXi4uIkScnJyXYsCQCohi2hf/HFF6u+HjBggB1LAABqye2h//jjj9WnTx998803mj17tpxOp0aOHKkGDRq4eykAQC249Rh9UlKS1q5dqytXrmjy5Mm6dOmSgoKClJCQ4M5lAAA3wa07+v3792vRokWqrKzU3//+d33yySfy8/NTVFSUO5cBANwEt+7ov32lzZ49e9ShQwf5+flJUtVZLAEAdc+tO/p69epp+/btSktL02OPPSZJ+uyzz675SEEAQN1y647+jTfeUGpqqpo1a6aoqCht27ZNiYmJmjhxojuXAQDcBFs+HPz74MPBcaviw8FxK7vRh4Pb8s5YAMCtg9ADgOFsCf3KlSuvubx06VI7lgEA1IJbX3Wzfv16paena8eOHcrMzJQkXblyRUeOHNHw4cPduRQAoJbcGvqf/OQnuvPOO3X+/HkNHTpUkuTl5aVWrVq5cxkAwE1w66GbgIAAdevWTQsXLlRpaan27Nmj8+fPKyQkxJ3LAABugi3H6JOTk5Wamipvb2+tWbNGiYmJdiwDAKgFW05TvGvXLq1YsUKS9Mtf/lJDhgyxYxkAQC3YsqOvrKyUy+WSdPVDSBwOhx3LAABqwZYdff/+/RUVFaUuXbpoz5496t+/vx3LAABqwZbQP/vss+rRo4eOHTumwYMHq0OHDnYsAwCoBbeGfs2aNd+57sCBAzpw4IAGDRrkzqUAALXk1tBnZWVdc9myLK1evVq+vr6EHgA8xK2hj4uLq/o6JydH8fHx6tmzpyZMmODOZQAAN8GWY/TLly/XkiVLNH78ePXq1cuOJQAAteTW0BcUFGj8+PEKCAjQypUrFRAQ4M6HBwD8G9wa+ieffFI+Pj7q3r273n777WtuS05OdudSAIBacmvo//SnP7nz4QAAbuDW0D/44IPufDgAgBvwCVMAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDjv6m7Yvn17tXfq0aOHLcMAANyv2tBv2LCh2jsRegD44ag29FOnTr3u9YWFhbYNAwBwv2pD/613331XKSkpunz5ssrKynTXXXfdcLcPALi11PhkbEZGhjIyMjRgwABt3LhRISEhdTEXAMBNagx9YGCgnE6nSkpK1KZNG5WWltbFXAAAN6kx9M2aNVNqaqr8/PyUnJys4uLiupgLAOAmDsuyrBt9g8vl0qlTpxQQEKC0tDRFRkaqXbt2tg20bli4bY8NfB8lYzZ7egSgWlERLau9rcYnY9etW1f1dePGjbVv3z5bQw8AcK8aQ5+VlSVJsixLBw8eVGBgoAYNGmT3XAAAN6kx9HFxcVVfW5alkSNH2joQAMC9agx9RUVF1denT5/WiRMnbB0IAOBeNYa+X79+cjgcsixLvr6+ev75520diCe8AMC9agz9zJkz1blz56rLO3futHUgAIB7VRv6f/zjHzp69KgWL16sZ555RtLVl1ouX75c69evr7MBAQDfT7Wh9/f3V1FRkSoqKnT69GlJksPh0JgxY+psOADA91fjG6YKCgp09uxZhYeH6+OPP9YjjzwiHx8f2wb64Aue7AWAm3WjN0zVeAqEKVOm6Msvv5QkZWdnKz4+3m2DAQDsV2PoCwoKFBUVJUkaMWIE56MHgB+YWn1mbHZ2tiQpJydHLpfL1oEAAO5V48srJ0yYoFGjRunMmTPy9fXVz372s7qYCwDgJjXu6Lt06aLJkyfroYceUmlpqc6cOVMXcwEA3KTaHX1FRYU2bNig5cuXy+l0qri4WFu2bJGvr29dzgcA+J6q3dE/+uijOnTokJKSkpSSkqKmTZsSeQD4Aap2Rz98+HCtX79eX3/9tQYPHqwaXm4PALhF1fiGqZ07d2rlypXKyMjQ4MGDNXDgQHXo0MG2gXjDFADcvBu9YarG0H/r4sWLWrt2rVatWqU1a9a4a7bvIPQAcPPcEvq6QugB4OZ9r1MgAAB+2Ag9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABjO29MDwD5z40eqvl9DSVJQ02Ya9KuxHp4IkL74+yZ9+cn/SJIqL1coP+eoXp+bKr+GjTw8mbkIvaEuV1RIkp558w8engS4VkTPforo2U+StGHhLEX07EfkbcahG0MV5GTpcnmZlk4Zq8WT45R35ICnRwKu8XXWIRXm5ahrnyc9PYrxbNvR5+XlaevWrSovL6+6bsSIEXYth3/hU7++HnpyiO5/tL/OnDqh5Ynj9esZS1SvXj1PjwZIkratSVHPwTGeHuO2YNuO/qWXXtKFCxfkdDqr/qDu3NG8pTr/pI8cDoeatGglv8b+Kj53xtNjAZKk0pJiFZ3MVei9EZ4e5bZg246+efPmeuWVV+x6eNTgi62bVJCXrSef+40uni1SeeklNQq6w9NjAZKknIN7FHbfjzw9xm3DttD36tVLSUlJateuXdV1gwYNsms5/IuIRx/Xmtm/14I3fyOHpIEjx3DYBreMM6fyFNS0uafHuG3YFvqNGzcqLCxMWVlZkiSHw2HXUrgOb28fDX71DU+PAVzXjwcM9fQItxXbQu90OvXWW2/Z9fAAgFqyLfQtWrTQn//8Z91zzz1Vu/kePXrYtRwAoBq2hb6yslLHjx/X8ePHq64j9ABQ92wL/dSpU6+5XFhYaNdSAIAbsC307777rlJSUnT58mWVlZXprrvu0oYNG+xaDgBQDdveMJWRkaGMjAwNGDBAGzduVEhIiF1LAQBuwLbQBwYGyul0qqSkRG3atFFpaaldSwEAbsC20Ddr1kypqany8/NTUlKSiouL7VoKAHADDsuyLDse2OVyKT8/X/7+/kpLS1NkZOQ175KtzgdfnLBjHAAwWlREy2pvs+3J2HPnzmnhwoU6fvy42rdvrzvvvNOupQAAN2DboZtRo0YpLCxMr7/+ulq2bKmxY/l0IwDwBFs/YWrYsGGSpI4dO2rTpk12LgUAqIZtO/qwsDCtXbtWBQUFSk9PV2BgoLKzs5WdnW3XkgCA67DtydiYmKufHHPx4kXVq1dPDRte/ZBqh8OhpUuXVns/nowFgJt3oydj3b6j379/vwYNGqQFCxYoJiZGp0+fVklJiWJjY7Vs2bIbRh4A4H5uD/2MGTOUmJgop9OpmTNnav78+Vq1apXmz5/v7qUAALXg9idjLctSx44dVVBQoNLSUt17772S+OARAPAUt+/oXS6XJGnbtm2KjIyUJFVUVOjSpUvuXgoAUAtu39FHRkbq6aefVn5+vubMmaPc3FwlJCSof//+7l4KAFALtrzqJisrS8HBwQoKClJubq4OHTqkvn371uq+vOoGAG5enZ8CoW3btlVft27dWq1bt7ZjGQBALdj2hikAwK2B0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABjOYVmW5ekhAAD2YUcPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOG9PD4CrduzYoZdfflkfffSRmjdvLklKSkpSWFiYnnrqqeve5/z589q2bZsGDBhwzfU5OTmaMmWKrly5osrKSnXq1ElxcXHy8uL/ddhr3rx5+uyzz+Tl5SWHw6HRo0erU6dOnh7rtse//FuIj4+Pxo8fr9q+teHQoUNKT0//zvV/+MMfFB0drQULFmjx4sU6fvy4tmzZ4u5xgWscPXpU6enpWrRokRYuXKjXX39dEyZM8PRYEDv6W0r37t3lcrm0fPlyRUdHX3PbwoULtWHDBnl7e6tr164aM2aM5s6dq6+++koffvihhg4dWvW9LVq0UFpamho2bKjOnTtr5syZ8vb21o4dOzR37lx5eXnp9OnTGjp0qH7xi19o586d+uMf/yhJKisr07Rp0+Tj46PRo0erefPmOnHihJ544gkdOXJEBw4cUM+ePfXaa6/V6e8Gt77g4GCdPHlSqampevjhhxUeHq7U1FTFxMQoNDRU2dnZsixLM2bMUHBwsH77298qPz9f586d08MPP6xRo0YpPj5e3t7eOnnypCoqKtS/f39t3bpVp06d0uzZs9W6dWtP/5g/TBZuCZmZmdaoUaOss2fPWr1797ays7Ot6dOnW6tWrbK++uora/DgwVZFRYXlcrmsl19+2UpPT6+6z78qLy+3Fi1aZA0bNszq2rWrFRcXZ124cMHKzMy0Hn/8cau8vNwqLS21+vTpYxUVFVnvv/++lZ+fb1mWZc2ZM8eaPXu2lZeXZ3Xr1s26ePGiVVhYaN13333WuXPnrLKyMisyMrKufz34gdi3b58VHx9vPfLII9ZPf/pTa9OmTVZ0dLSVlpZmWZZlvf/++9bkyZOtvLw86y9/+YtlWZZVVlZmPfjgg5ZlWda4ceOs2bNnW5ZlWZMmTbKmTZtmWZZlzZo1y1q0aFGd/zymYEd/iwkKCtKECRMUHx+v+++/X5J07NgxdenSRT4+PpKkrl276siRI+rSpct1HyMzM1OxsbGKjY1VSUmJpk2bptmzZ6tXr16KiIiQ0+mUJLVv3165ubkKCQnRlClT1KBBAxUUFFSt26pVKzVu3FhOp1NNmjRRYGCgJMnhcNj8W8APUU5Ojho1aqSpU6dKkvbu3asXXnhBTZo0Uffu3SVJ999/v9LT0xUYGKi9e/cqMzNTjRo1UkVFRdXj3HPPPZIkf39/hYWFVX39z9+Dm8Mx+lvQo48+qtDQUKWlpUmSwsLCtGfPHlVWVsqyLO3atUuhoaHy8vKSy+X6zv2nT5+uTz/9VJLUsGFDhYaGVsX94MGDunLlikpLS3X06FG1adNGEydO1DvvvKPExEQ1bdq06jkCgo6bcejQISUkJKi8vFySFBoaqsaNG6tevXrat2+fJOnzzz9Xu3bttHr1ajVu3FjJycl69tlnVVZWxt87G7Gjv0W98cYbyszMlCTdfffdevzxxxUVFSWXy6Uf/ehH6tOnjwoLC3X48GEtXrxYsbGxVfedOXOmfve73yk5OVlOp1MtW7ZUQkKC9u/fr8rKSo0YMULnz5/Xr371KwUHB2vgwIEaMmSI/P391aRJExUWFnrop8YP2WOPPaasrCz9/Oc/V4MGDWRZlsaOHaslS5YoLS1Nixcvlp+fn37/+9+rqKhIr732mnbv3i0/Pz+1adOGv3c24uyVt5EdO3ZoxYoVmjFjhqdHwW0kJiZGCQkJatu2radHuW1x6AYADMeOHgAMx44eAAxH6AHAcIQeAAxH6GGsHTt2KDIyUjExMYqJidGQIUO0bNmym36cpKQkrV69WgcPHqw6VcT1bN68WQUFBbV6zIyMDMXHx9/0LMC/g9fRw2jdu3evejlpRUWF+vXrp4EDB8rf3/+mHys8PFzh4eHV3r506VIlJCQoJCTk354XsAOhx22juLhYXl5eio2NVcuWLXXx4kXNmzdPCQkJysnJkcvl0qhRo9StWzf97W9/05w5cxQcHKzLly8rLCzsmvchrFy5Uh988IFcLpd69+6t++67TwcPHtS4ceOUkpKiDz/8UOvXr5fD4VD//v01fPhwZWVlacKECfLz85Ofn58CAgI8/SvBbYLQw2iZmZmKiYmRw+GQj4+PJk2apPfee08DBgxQ3759lZKSoqCgIL3zzjs6d+6coqOjtWHDBk2fPl0rV65UYGCgXnjhhWse88yZM5o/f77WrVsnp9OpxMREPfDAAwoPD1dCQoJyc3O1ceNGpaSkyOFwKDY2Vj169NCsWbP06quv6sc//rHmzZunY8eOeei3gtsNoYfR/vnQzbfee+89hYaGSpIOHz6s3bt3a8+ePZKkyspKFRUVqVGjRgoKCpIkRUREXHP/vLw8tW/fXr6+vpL0nXOuHz58WCdPnqw6LcWFCxeUm5urI0eOqHPnzpKuntyL0KOu8GQsbkvfnjgrLCxMTzzxhJYtW6b58+erX79+8vf31zfffKOzZ89KunoWxn/WunVrHTt2rOpsiq+++qoKCgrkcDhkWZbCwsLUrl07LV26VMuWLdNTTz2lDh06KCwsTF988YUkVZ3kC6gL7OhxW3v66ac1ceJERUdHq7i4WMOGDZPT6dTUqVP13HPPKSAgQN7e1/4zCQ4O1ogRIxQdHS2Hw6FevXopJCREERERGjt2rBYuXKjIyEhFRUWpoqJCnTt3VkhIiN58802NHj1aCxYsUHBwsOrXr++hnxq3G06BAACG49ANABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4f4fIK4U5wCYo6MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "confusion_matrix = pd.crosstab(y_test, pred, rownames=['Actual'], colnames=['Predicted'])\n",
    "plt.figure(figsize = (6, 6))\n",
    "sns.heatmap(confusion_matrix, annot = True, cmap = 'Paired', cbar = False, fmt=\"d\", xticklabels=['Not Spam', 'Spam'], yticklabels=['Not Spam', 'Spam']);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "360a61d0",
   "metadata": {},
   "source": [
    "## **Random Forest Classifier**<br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "e23fcdbb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 793 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "rfc = RandomForestClassifier()\n",
    "rfc.fit(dtv, y_train)\n",
    "pred = rfc.predict(test_dtv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "ef57db24",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  96.42857142857143\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy: ', accuracy_score(y_test, pred) * 100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36abe086",
   "metadata": {},
   "source": [
    "**Classification Report**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "9910630b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       184\n",
      "           1       0.69      0.75      0.72        12\n",
      "\n",
      "    accuracy                           0.96       196\n",
      "   macro avg       0.84      0.86      0.85       196\n",
      "weighted avg       0.97      0.96      0.96       196\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26682407",
   "metadata": {},
   "source": [
    "**Confusion Matrix**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "89c33917",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAFxCAYAAAB9Z2NTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAZ6ElEQVR4nO3de1jUdf738dcgjOABkCzU20PgIWlNf7SasrKWqa1Z/nS7TMMFY2vNXdu6UFPRbKPMxFXSunfV1TwfsgVFTV27TS2sFnW1fh7zgAc8JOAJA0EY53v/4RXX+isEtvky+uH5uC6va5hh5vOGS599mpnvdxyWZVkCABjLx9sDAADsRegBwHCEHgAMR+gBwHCEHgAMR+gBwHC+3h7gf1s7OMLbIwA/qnD0Jm+PAJQrJrJpubexowcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoTdEcMv2+sWERZKkwBZtFf3GCnV9fan+64W3JIdDktS8+9Pq9laqot9YodDIR7w3LCCpIP+S3hn+jPLOZHt7FOP5ensA/HStnnxeTaP/W65rRZKk+556UYfTZyr36ww9+OKfFRr5sC5n7VP4r2KVMWGAfPxqK/r1Zcrb+4XcrlIvT4+a6LrLpXXvT5ev0+ntUWoEdvQGKMzJ1s4ZL5d9nX/ioJx1gyRJvv515Xa5FNzyAV08vFtuV6lcRQUqzMlWYPP7vDUyarj/t3S2Ovbsq/oNGnp7lBrBltBPnz5dXbt2VXR0dNkf2OfbnZtu2pkXnjuhds+OV/dp61U76C5dOLhDfgH1VHq1oOx7XMWF8q1T3xvjoob76tONqhMYrFYdOnl7lBrDlqduPvvsM23dulVO/rfMK9oNGa8v3ojTd2eO6t5eg/Wz34xV7p7P5RtQt+x7fP3rqrTwihenRE311acb5XA4dGzvbp07eVTpM5MVM/ot1Q8O8fZoxrIl9BEREbp27Rqh95KSwnyVFt3YvRdfylVIm0hdztqriIEJ8vFzysfXqXr/J1zfnT7i5UlREz2XNKPs8oI3RurJ3yUQeZvZEvrWrVsrOjpaDRs2lGVZcjgc2rx5sx1L4Uf8z9zX1PGlFLnd12W5SvX13Nd0Lf+8jn28VF3/tFQOHx998+EMuUtLvD0qgGrgsCzL8vSDDhgwQLNnz1ZgYGDZdZXd3a8dHOHpcQCPKBy9ydsjAOWKiWxa7m227OibNGmigIAAnroBgNuALaE/d+6cevXqpWbNmkmSHA6HVqxYYcdSAIAK2BL66dOn2/GwAID/gC2hd7lc2rhxo0pLb7y3Ozc3V2+++aYdSwEAKmDLAVNjx46VJO3evVunT5/W5cuX7VgGAFAJtoTe399fw4YNU2hoqJKTk3X+/Hk7lgEAVIItobcsS3l5eSosLNTVq1eVn59vxzIAgEqwJfR//OMftWnTJvXr1089evRQt27d7FgGAFAJtrwY26lTJ7Vs2VKnTp3SP/7xDwUHB9uxDACgEmwJ/bJly7Ro0SK1bt1aR48e1fDhw9WvXz87lgIAVMCW0Kempuqjjz5S7dq1VVRUpNjYWEIPAF5iy3P0d911l2rVqiXpxjtweOoGALzHlh29ZVnq37+/IiMjdeDAAblcLo0aNUqSlJKSYseSAIBy2BL63//+92WX+/bta8cSAIBK8njoP/nkE/Xs2VPfffedZs6cKafTqWHDhqlOnTqeXgoAUAkefY5+2rRpWrNmja5fv66JEyfq6tWratCggZKSkjy5DACgCjy6o9+/f78WLFggl8ulTz/9VJ999pkCAgIUExPjyWUAAFXg0R399++02bNnj9q0aaOAgABJKjuLJQCg+nl0R1+rVi19/vnnSk9P12OPPSZJ+vLLL2/6SEEAQPXy6I7+1VdfVVpamho1aqSYmBht27ZNycnJmjBhgieXAQBUgS0fDv5T8OHguF3x4eC4nd3qw8FtOTIWAHD7IPQAYDhbQp+amnrT14sXL7ZjGQBAJXj0XTfr1q3Tli1btH37dmVmZkqSrl+/riNHjmjIkCGeXAoAUEkeDf0vf/lL3X333bp8+bIGDRokSfLx8VGzZs08uQwAoAo8+tRNUFCQOnfurPnz56uoqEh79uzR5cuXFRoa6sllAABVYMtz9CkpKUpLS5Ovr69Wr16t5ORkO5YBAFSCLacp3rlzp1asWCFJevbZZzVw4EA7lgEAVIItO3qXyyW32y3pxoeQOBwOO5YBAFSCLTv6Pn36KCYmRh06dNCePXvUp08fO5YBAFSCLaF/7rnnFB0drWPHjmnAgAFq06aNHcsAACrBo6FfvXr1D647cOCADhw4oP79+3tyKQBAJXk09FlZWTd9bVmWVq1aJX9/f0IPAF7i0dCPGjWq7PLJkyeVmJioRx55ROPHj/fkMgCAKrDlOfply5Zp0aJFGjdunLp3727HEgCASvJo6HNycjRu3DgFBQUpNTVVQUFBnnx4AMB/wKOhf/LJJ+Xn56cuXbrozTffvOm2lJQUTy4FAKgkj4b+r3/9qycfDgDgAR4N/UMPPeTJhwMAeACfMAUAhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhvMt74bPP/+83DtFR0fbMgwAwPPKDf369evLvROhB4A7R7mhnzx58o9en5uba9swAADPKzf033vvvfe0fPlylZaWqri4WPfee+8td/sAgNtLhS/GZmRkKCMjQ3379tWGDRsUGhpaHXMBADykwtAHBwfL6XSqsLBQLVq0UFFRUXXMBQDwkApD36hRI6WlpSkgIEApKSkqKCiojrkAAB7isCzLutU3uN1uffvttwoKClJ6erqioqLUqlUr2wZaOzjCtscGforC0Zu8PQJQrpjIpuXeVuGLsWvXri27XL9+fe3bt8/W0AMAPKvC0GdlZUmSLMvSwYMHFRwcrP79+9s9FwDAQyoM/ahRo8ouW5alYcOG2ToQAMCzKgx9SUlJ2eW8vDydPn3a1oEAAJ5VYeh79+4th8Mhy7Lk7++v3/3ud7YOxAteAOBZFYZ+xowZat++fdnXO3bssHUgAIBnlRv6f/3rXzp69KgWLlyo3/72t5JuvNVy2bJlWrduXbUNCAD4acoNfWBgoM6fP6+SkhLl5eVJkhwOh0aPHl1twwEAfroKD5jKycnRxYsXFRERoU8++UQPP/yw/Pz8bBvog694sRcAqupWB0xVeAqESZMm6euvv5YkHT9+XImJiR4bDABgvwpDn5OTo5iYGEnS0KFDOR89ANxhKvWZscePH5cknTx5Um6329aBAACeVeHbK8ePH6+EhARduHBB/v7++vWvf10dcwEAPKTCHX2HDh00ceJE/eIXv1BRUZEuXLhQHXMBADyk3B19SUmJ1q9fr2XLlsnpdKqgoECbN2+Wv79/dc4HAPiJyt3RP/roozp06JCmTZum5cuX65577iHyAHAHKndHP2TIEK1bt05nzpzRgAEDVMHb7QEAt6kKD5jasWOHUlNTlZGRoQEDBqhfv35q06aNbQNxwBQAVN2tDpiqMPTfu3LlitasWaOVK1dq9erVnprtBwg9AFSdR0JfXQg9AFTdTzoFAgDgzkboAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwvt4eAPZwu69r7Zx3dOHsKTl8fNT/92MU0qiJt8cC5Cot0epZU3Up91vVDqijJ557WXc1burtsYzGjt5Qh3b9U5L0/JvvqfvT8fp4ySwvTwTcsGvzBjn9AzT0rb+oz29f0oYF/9fbIxmPHb2hIjpFq82DUZKk/PM5qhvUwMsTATfknTmp1v/1kCSpYZNmyjuT7eWJzGdb6E+dOqWtW7fq2rVrZdcNHTrUruXwI2rVqqX0mck6uPMLDUx43dvjAJKkRi1a6vDuf6ptp646ffSgvrt4Xm73dfn41PL2aMay7amb4cOHKz8/X06ns+wPqt+vhyfqpemL9NHcFJUUF3l7HECR3R9X7YC6WvjmKB3e9U81Dm9N5G1m246+cePGeumll+x6eFTgfzI26crFPP2y/2D5OWvL4fCRg39MuA2czfpGzdu2U+9nh+tM1iFdzDnr7ZGM57Asy7LjgT/44AOdOXNGrVq1Kruuf//+Fd/vq9N2jFPjlBQXafXsqSq4fFHu6y5F94tR245dvT0WoMIr+Up77y2VXiuSf516+u9hrygwpKG3x7rjxUSW/84l20IfFxen8PBwBQYG3ljI4dDIkSMrvB+hB4Cqu1XobXvqxul06o033rDr4QEAlWRb6Js0aaK//e1vuv/+++VwOCRJ0dHRdi0HACiHbaF3uVw6ceKETpw4UXYdoQeA6mdb6CdPnnzT17m5uXYtBQC4BdtC/95772n58uUqLS1VcXGx7r33Xq1fv96u5QAA5bDtgKmMjAxlZGSob9++2rBhg0JDQ+1aCgBwC7aFPjg4WE6nU4WFhWrRooWKijgqEwC8wbbQN2rUSGlpaQoICNC0adNUUFBg11IAgFuw7YApt9utc+fOKTAwUOnp6YqKirrpKNnycMAUAFSdVw6YunTpkubPn68TJ06odevWuvvuu+1aCgBwC7Y9dZOQkKDw8HC98soratq0qcaMGWPXUgCAW7D1g0cGDx4sSWrbtq02btxo51IAgHLYtqMPDw/XmjVrlJOToy1btig4OFjHjx/X8ePH7VoSAPAjbD17pSRduXJFtWrVUt26dW8s6HBo8eLF5d6PF2MBoOpu9WKsx3f0+/fvV//+/TVv3jzFxcUpLy9PhYWFio+P15IlS24ZeQCA53k89NOnT1dycrKcTqdmzJihuXPnauXKlZo7d66nlwIAVILHX4y1LEtt27ZVTk6OioqK9LOf/UySyk5VDACoXh7f0bvdbknStm3bFBUVJUkqKSnR1atXPb0UAKASPL6jj4qK0jPPPKNz585p1qxZys7OVlJSkvr06ePppQAAlWDLu26ysrIUEhKiBg0aKDs7W4cOHVKvXr0qdV/edQMAVVftp0Bo2bJl2eXmzZurefPmdiwDAKgE2w6YAgDcHgg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4RyWZVneHgIAYB929ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIbz9fYAuGH79u168cUX9dFHH6lx48aSpGnTpik8PFxPPfXUj97n8uXL2rZtm/r27XvT9SdPntSkSZN0/fp1uVwutWvXTqNGjZKPD/9dh73mzJmjL7/8Uj4+PnI4HBoxYoTatWvn7bFqPP7l30b8/Pw0btw4VfbQhkOHDmnLli0/uP6dd95RbGys5s2bp4ULF+rEiRPavHmzp8cFbnL06FFt2bJFCxYs0Pz58/XKK69o/Pjx3h4LYkd/W+nSpYvcbreWLVum2NjYm26bP3++1q9fL19fX3Xs2FGjR4/W7Nmz9c033+jDDz/UoEGDyr63SZMmSk9PV926ddW+fXvNmDFDvr6+2r59u2bPni0fHx/l5eVp0KBB+s1vfqMdO3boL3/5iySpuLhYU6ZMkZ+fn0aMGKHGjRvr9OnTeuKJJ3TkyBEdOHBAjzzyiEaOHFmtvxvc/kJCQnT27FmlpaWpW7duioiIUFpamuLi4hQWFqbjx4/LsixNnz5dISEh+tOf/qRz587p0qVL6tatmxISEpSYmChfX1+dPXtWJSUl6tOnj7Zu3apvv/1WM2fOVPPmzb39Y96ZLNwWMjMzrYSEBOvixYtWjx49rOPHj1tTp061Vq5caX3zzTfWgAEDrJKSEsvtdlsvvviitWXLlrL7/G/Xrl2zFixYYA0ePNjq2LGjNWrUKCs/P9/KzMy0Hn/8cevatWtWUVGR1bNnT+v8+fPW0qVLrXPnzlmWZVmzZs2yZs6caZ06dcrq3LmzdeXKFSs3N9d64IEHrEuXLlnFxcVWVFRUdf96cIfYt2+flZiYaD388MPWr371K2vjxo1WbGyslZ6eblmWZS1dutSaOHGiderUKevvf/+7ZVmWVVxcbD300EOWZVnW2LFjrZkzZ1qWZVmvvfaaNWXKFMuyLOvdd9+1FixYUO0/jynY0d9mGjRooPHjxysxMVEPPvigJOnYsWPq0KGD/Pz8JEkdO3bUkSNH1KFDhx99jMzMTMXHxys+Pl6FhYWaMmWKZs6cqe7duysyMlJOp1OS1Lp1a2VnZys0NFSTJk1SnTp1lJOTU7Zus2bNVL9+fTmdTjVs2FDBwcGSJIfDYfNvAXeikydPql69epo8ebIkae/evXrhhRfUsGFDdenSRZL04IMPasuWLQoODtbevXuVmZmpevXqqaSkpOxx7r//fklSYGCgwsPDyy7/+/eganiO/jb06KOPKiwsTOnp6ZKk8PBw7dmzRy6XS5ZlaefOnQoLC5OPj4/cbvcP7j916lR98cUXkqS6desqLCysLO4HDx7U9evXVVRUpKNHj6pFixaaMGGC3n77bSUnJ+uee+4pe42AoKMqDh06pKSkJF27dk2SFBYWpvr166tWrVrat2+fJGn37t1q1aqVVq1apfr16yslJUXPPfeciouL+XtnI3b0t6lXX31VmZmZkqT77rtPjz/+uGJiYuR2u/Xzn/9cPXv2VG5urg4fPqyFCxcqPj6+7L4zZszQW2+9pZSUFDmdTjVt2lRJSUnav3+/XC6Xhg4dqsuXL+sPf/iDQkJC1K9fPw0cOFCBgYFq2LChcnNzvfRT40722GOPKSsrS08//bTq1Kkjy7I0ZswYLVq0SOnp6Vq4cKECAgL05z//WefPn9fIkSO1a9cuBQQEqEWLFvy9sxFnr6xBtm/frhUrVmj69OneHgU1SFxcnJKSktSyZUtvj1Jj8dQNABiOHT0AGI4dPQAYjtADgOEIPQAYjtDDWNu3b1dUVJTi4uIUFxengQMHasmSJVV+nGnTpmnVqlU6ePBg2akifsymTZuUk5NTqcfMyMhQYmJilWcB/hO8jx5G69KlS9nbSUtKStS7d2/169dPgYGBVX6siIgIRURElHv74sWLlZSUpNDQ0P94XsAOhB41RkFBgXx8fBQfH6+mTZvqypUrmjNnjpKSknTy5Em53W4lJCSoc+fO+vjjjzVr1iyFhISotLRU4eHhNx2HkJqaqg8++EBut1s9evTQAw88oIMHD2rs2LFavny5PvzwQ61bt04Oh0N9+vTRkCFDlJWVpfHjxysgIEABAQEKCgry9q8ENQShh9EyMzMVFxcnh8MhPz8/vfbaa3r//ffVt29f9erVS8uXL1eDBg309ttv69KlS4qNjdX69es1depUpaamKjg4WC+88MJNj3nhwgXNnTtXa9euldPpVHJysjp16qSIiAglJSUpOztbGzZs0PLly+VwOBQfH6/o6Gi9++67evnll9W1a1fNmTNHx44d89JvBTUNoYfR/v2pm++9//77CgsLkyQdPnxYu3bt0p49eyRJLpdL58+fV7169dSgQQNJUmRk5E33P3XqlFq3bi1/f39J+sE51w8fPqyzZ8+WnZYiPz9f2dnZOnLkiNq3by/pxsm9CD2qCy/Gokb6/sRZ4eHheuKJJ7RkyRLNnTtXvXv3VmBgoL777jtdvHhR0o2zMP675s2b69ixY2VnU3z55ZeVk5Mjh8Mhy7IUHh6uVq1aafHixVqyZImeeuoptWnTRuHh4frqq68kqewkX0B1YEePGu2ZZ57RhAkTFBsbq4KCAg0ePFhOp1OTJ0/W888/r6CgIPn63vzPJCQkREOHDlVsbKwcDoe6d++u0NBQRUZGasyYMZo/f76ioqIUExOjkpIStW/fXqGhoXr99dc1YsQIzZs3TyEhIapdu7aXfmrUNJwCAQAMx1M3AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8Ahvv/d7U2xhfYmF0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "confusion_matrix = pd.crosstab(y_test, pred, rownames=['Actual'], colnames=['Predicted'])\n",
    "plt.figure(figsize = (6, 6))\n",
    "sns.heatmap(confusion_matrix, annot = True, cmap = 'Paired', cbar = False, fmt=\"d\", xticklabels=['Not Spam', 'Spam'], yticklabels=['Not Spam', 'Spam']);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc7d3564",
   "metadata": {},
   "source": [
    "## **Multinomial Naive Bayes** <br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "b8cb28ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 54.9 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "mnb = MultinomialNB()\n",
    "mnb.fit(dtv, y_train)\n",
    "pred = mnb.predict(test_dtv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "926b89e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  95.91836734693877\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy: ', accuracy_score(y_test, pred) * 100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2bcb03d",
   "metadata": {},
   "source": [
    "**Classification Report**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "540d0838",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       184\n",
      "           1       0.67      0.67      0.67        12\n",
      "\n",
      "    accuracy                           0.96       196\n",
      "   macro avg       0.82      0.82      0.82       196\n",
      "weighted avg       0.96      0.96      0.96       196\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3aaad01b",
   "metadata": {},
   "source": [
    "**Confusion Matrix**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "f3821c9e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAFxCAYAAAB9Z2NTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAZsUlEQVR4nO3de1TVdb7/8ddG2ILKRbJQj5fES9GUHhpNmRjL1MYsj07HUXHAYSpzpqaOl1Q0nTAzcYS0fjPqeMG72YCipo5zTCysDtqUHa95VzTj4gUVBGGzv78/XLHyFILT/rL14/Oxlmtt9mZ/P29Y+uzTvny3w7IsSwAAY/l4ewAAgL0IPQAYjtADgOEIPQAYjtADgOEIPQAYztfbA/xf6wZHeHsE4AcVj97s7RGAKsVENqvyNnb0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0hghp3V4/m7BYkhTU8l5FT1qph19bpn9//g3J4ZAktej2K3V9I03Rk1YqLPJR7w0LSCq6cF5vvTBIBV/neHsU4/l6ewD8eG2eelbNov9DrislkqR7nn5RBzNmKf/LLD344p8UFvmICo/sUfgvYpU1ob98/Ooq+rXlKtj9idyuci9Pj9tRhcul9fNnyNfp9PYotwV29AYozsvRZzNfrvz6wvH9ctYPliT5+teX2+VSSOsHdO7gF3K7yuUqKVJxXo6CWtzjrZFxm/vvZXPUsUcfBTZs5O1Rbgu2hH7GjBl6+OGHFR0dXfkH9vnms83X7MyLc4/r/t+MV7fkDaobfIfO7t8hv4AGKr9cVPk9rtJi+dYL9Ma4uM3t/HCT6gWFqE2HTt4e5bZhy0M3H330kbZu3Son/1vmFfcPGa9PJsXp0teHdXfPwfrJr8cqf9fH8g2oX/k9vv71VV580YtT4na188NNcjgcOrr7C+WeOKyMWUmKGf2GAkNCvT2asWwJfUREhK5cuULovaSs+ILKS67u3kvP5yu0XaQKj+xWxIDh8vFzysfXqQb/Fq5Lpw55eVLcjp5JnFl5eeGkkXrqueFE3ma2hL5t27aKjo5Wo0aNZFmWHA6HtmzZYsdS+AH/O2+iOr6UIre7QparXF/Om6grF87o6D+W6eE/LpPDx0dfvTdT7vIyb48KoBY4LMuyPH3Q/v37a86cOQoKCqq8rqa7+3WDIzw9DuARxaM3e3sEoEoxkc2qvM2WHX3Tpk0VEBDAQzcAcBOwJfS5ubnq2bOnmjdvLklyOBxauXKlHUsBAKphS+hnzJhhx2EBAP8CW0Lvcrm0adMmlZdffW13fn6+Xn/9dTuWAgBUw5Y3TI0dO1aS9MUXX+jUqVMqLCy0YxkAQA3YEnp/f38NGzZMYWFhSkpK0pkzZ+xYBgBQA7aE3rIsFRQUqLi4WJcvX9aFCxfsWAYAUAO2hP4Pf/iDNm/erL59+6p79+7q2rWrHcsAAGrAlidjO3XqpNatW+vkyZP6+9//rpCQEDuWAQDUgC2hX758uRYvXqy2bdvq8OHDeuGFF9S3b187lgIAVMOW0Kelpen9999X3bp1VVJSotjYWEIPAF5iy2P0d9xxh+rUqSPp6itweOgGALzHlh29ZVnq16+fIiMjtW/fPrlcLo0aNUqSlJKSYseSAIAq2BL63/3ud5WX+/TpY8cSAIAa8njoP/jgA/Xo0UOXLl3SrFmz5HQ6NWzYMNWrV8/TSwEAasCjj9EnJydr7dq1qqio0OTJk3X58mU1bNhQiYmJnlwGAHADPLqj37t3rxYuXCiXy6UPP/xQH330kQICAhQTE+PJZQAAN8CjO/pvX2mza9cutWvXTgEBAZJUeRZLAEDt8+iOvk6dOvr444+VkZGhxx9/XJL06aefXvORggCA2uXRHf2rr76q9PR0NW7cWDExMdq2bZuSkpI0YcIETy4DALgBtnw4+I/Bh4PjZsWHg+Nmdr0PB7flnbEAgJsHoQcAw9kS+rS0tGu+XrJkiR3LAABqwKOvulm/fr0yMzO1fft2ZWdnS5IqKip06NAhDRkyxJNLAQBqyKOh//nPf64777xThYWFGjhwoCTJx8dHzZs39+QyAIAb4NGHboKDg9W5c2elpqaqpKREu3btUmFhocLCwjy5DADgBtjyGH1KSorS09Pl6+urNWvWKCkpyY5lAAA1YMtpij/77DOtXLlSkvSb3/xGAwYMsGMZAEAN2LKjd7lccrvdkq5+CInD4bBjGQBADdiyo+/du7diYmLUoUMH7dq1S71797ZjGQBADdgS+meeeUbR0dE6evSo+vfvr3bt2tmxDACgBjwa+jVr1nzvun379mnfvn3q16+fJ5cCANSQR0N/5MiRa762LEurV6+Wv78/oQcAL/Fo6EeNGlV5+cSJE0pISNCjjz6q8ePHe3IZAMANsOUx+uXLl2vx4sUaN26cunXrZscSAIAa8mjo8/LyNG7cOAUHBystLU3BwcGePDwA4F/g0dA/9dRT8vPzU5cuXfT6669fc1tKSoonlwIA1JBHQ/+Xv/zFk4cDAHiAR0P/0EMPefJwAAAP4BOmAMBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwvlXd8PHHH1d5p+joaFuGAQB4XpWh37BhQ5V3IvQAcOuoMvRTp079wevz8/NtGwYA4HlVhv5b77zzjlasWKHy8nKVlpbq7rvvvu5uHwBwc6n2ydisrCxlZWWpT58+2rhxo8LCwmpjLgCAh1Qb+pCQEDmdThUXF6tly5YqKSmpjbkAAB5SbegbN26s9PR0BQQEKCUlRUVFRbUxFwDAQxyWZVnX+wa3261vvvlGwcHBysjIUFRUlNq0aWPbQOsGR9h2bODHKB692dsjAFWKiWxW5W3VPhm7bt26ysuBgYHas2ePraEHAHhWtaE/cuSIJMmyLO3fv18hISHq16+f3XMBADyk2tCPGjWq8rJlWRo2bJitAwEAPKva0JeVlVVeLigo0KlTp2wdCADgWdWGvlevXnI4HLIsS/7+/nruuedsHYgnvADAs6oN/cyZM9W+ffvKr3fs2GHrQAAAz6oy9P/85z91+PBhLVq0SL/97W8lXX2p5fLly7V+/fpaGxAA8ONUGfqgoCCdOXNGZWVlKigokCQ5HA6NHj261oYDAPx41b5hKi8vT+fOnVNERIQ++OADPfLII/Lz87NtoHd38mQvANyo671hqtpTIEyZMkVffvmlJOnYsWNKSEjw2GAAAPtVG/q8vDzFxMRIkoYOHcr56AHgFlOjz4w9duyYJOnEiRNyu922DgQA8KxqX145fvx4DR8+XGfPnpW/v79++ctf1sZcAAAPqXZH36FDB02ePFk/+9nPVFJSorNnz9bGXAAAD6lyR19WVqYNGzZo+fLlcjqdKioq0pYtW+Tv71+b8wEAfqQqd/SPPfaYDhw4oOTkZK1YsUJ33XUXkQeAW1CVO/ohQ4Zo/fr1+vrrr9W/f39V83J7AMBNqto3TO3YsUNpaWnKyspS//791bdvX7Vr1862gXjDFADcuOu9Yara0H/r4sWLWrt2rVatWqU1a9Z4arbvIfQAcOM8EvraQugB4Mb9qFMgAABubYQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6A1WdOG83nphkAq+zvH2KEClCpdL6e9M0fyJLyn1tf/i72ctIPSGqnC5tH7+DPk6nd4eBbjGoS+3y+2u0HOT/58e+c84Zb6X6u2RjEfoDfXfy+aoY48+CmzYyNujANe4o0kzuSsq5Ha7daXksnzq1PH2SMbztevAJ0+e1NatW3XlypXK64YOHWrXcviOnR9uUr2gELXp0Enb1rzr7XGAazjrBqiwIFd/Hhmvy5cuavCYKd4eyXi27ehfeOEFXbhwQU6ns/IPasfODzfp6O7PtXDSSOWeOKyMWUm6VHjO22MBkqT/2ZiuNh066eWZS/T7aXO1ZvY0lZeVeXsso9m2o2/SpIleeukluw6P63gmcWbl5YWTRuqp54YrMCTUewMB3xFQP1A+vlfTE9AgUBUulyx3hZenMpttoe/WrZuSk5PVpk2byuv69etn13IAbhFdnuyvtXOmK/W1/1KFy6Xug56V0z/A22MZzWFZlmXHgePi4hQeHq6goKCrCzkcGjlyZLX3e3fnKTvGAQCjxUQ2q/I223b0TqdTkyZNsuvwAIAasi30TZs21V//+lfdd999cjgckqTo6Gi7lgMAVMG20LtcLh0/flzHjx+vvI7QA0Dtsy30U6dOvebr/Px8u5YCAFyHbaF/5513tGLFCpWXl6u0tFR33323NmzYYNdyAIAq2PaGqaysLGVlZalPnz7auHGjwsLC7FoKAHAdtoU+JCRETqdTxcXFatmypUpKSuxaCgBwHbaFvnHjxkpPT1dAQICSk5NVVFRk11IAgOuw7Q1Tbrdbubm5CgoKUkZGhqKioq55l2xVeMMUANw4r7xh6vz580pNTdXx48fVtm1b3XnnnXYtBQC4Dtseuhk+fLjCw8P1yiuvqFmzZhozZoxdSwEArsO2Hb0kDR48WJJ07733atOmTXYuBQCogm07+vDwcK1du1Z5eXnKzMxUSEiIjh07pmPHjtm1JADgB9h69kpJunjxourUqaP69etfXdDh0JIlS6q8H0/GAsCNu96TsR7f0e/du1f9+vXTggULFBcXp4KCAhUXFys+Pl5Lly69buQBAJ7n8dDPmDFDSUlJcjqdmjlzpubNm6dVq1Zp3rx5nl4KAFADHn8y1rIs3XvvvcrLy1NJSYl+8pOfSFLlqYoBALXL4zt6t9stSdq2bZuioqIkSWVlZbp8+bKnlwIA1IDHd/RRUVEaNGiQcnNzNXv2bOXk5CgxMVG9e/f29FIAgBqw5VU3R44cUWhoqBo2bKicnBwdOHBAPXv2rNF9edUNANy4Wj8FQuvWrSsvt2jRQi1atLBjGQBADdj2hikAwM2B0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABjOYVmW5e0hAAD2YUcPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOF9vD4Crtm/frhdffFHvv/++mjRpIklKTk5WeHi4nn766R+8T2FhobZt26Y+ffpcc/2JEyc0ZcoUVVRUyOVy6f7779eoUaPk48N/12GvuXPn6tNPP5WPj48cDodGjBih+++/39tj3fb4l38T8fPz07hx41TTtzYcOHBAmZmZ37v+rbfeUmxsrBYsWKBFixbp+PHj2rJli6fHBa5x+PBhZWZmauHChUpNTdUrr7yi8ePHe3ssiB39TaVLly5yu91avny5YmNjr7ktNTVVGzZskK+vrzp27KjRo0drzpw5+uqrr/Tee+9p4MCBld/btGlTZWRkqH79+mrfvr1mzpwpX19fbd++XXPmzJGPj48KCgo0cOBA/frXv9aOHTv05z//WZJUWlqqadOmyc/PTyNGjFCTJk106tQpPfnkkzp06JD27dunRx99VCNHjqzV3w1ufqGhoTp9+rTS09PVtWtXRUREKD09XXFxcWrVqpWOHTsmy7I0Y8YMhYaG6o9//KNyc3N1/vx5de3aVcOHD1dCQoJ8fX11+vRplZWVqXfv3tq6dau++eYbzZo1Sy1atPD2j3lrsnBTyM7OtoYPH26dO3fO6t69u3Xs2DFr+vTp1qpVq6yvvvrK6t+/v1VWVma53W7rxRdftDIzMyvv839duXLFWrhwoTV48GCrY8eO1qhRo6wLFy5Y2dnZ1hNPPGFduXLFKikpsXr06GGdOXPGWrZsmZWbm2tZlmXNnj3bmjVrlnXy5Emrc+fO1sWLF638/HzrgQcesM6fP2+VlpZaUVFRtf3rwS1iz549VkJCgvXII49Yv/jFL6xNmzZZsbGxVkZGhmVZlrVs2TJr8uTJ1smTJ62//e1vlmVZVmlpqfXQQw9ZlmVZY8eOtWbNmmVZlmVNnDjRmjZtmmVZlvX2229bCxcurPWfxxTs6G8yDRs21Pjx45WQkKAHH3xQknT06FF16NBBfn5+kqSOHTvq0KFD6tChww8eIzs7W/Hx8YqPj1dxcbGmTZumWbNmqVu3boqMjJTT6ZQktW3bVjk5OQoLC9OUKVNUr1495eXlVa7bvHlzBQYGyul0qlGjRgoJCZEkORwOm38LuBWdOHFCDRo00NSpUyVJu3fv1vPPP69GjRqpS5cukqQHH3xQmZmZCgkJ0e7du5Wdna0GDRqorKys8jj33XefJCkoKEjh4eGVl7/7PbgxPEZ/E3rsscfUqlUrZWRkSJLCw8O1a9cuuVwuWZalzz77TK1atZKPj4/cbvf37j99+nR98sknkqT69eurVatWlXHfv3+/KioqVFJSosOHD6tly5aaMGGC3nzzTSUlJemuu+6qfI6AoONGHDhwQImJibpy5YokqVWrVgoMDFSdOnW0Z88eSdIXX3yhNm3aaPXq1QoMDFRKSoqeeeYZlZaW8vfORuzob1KvvvqqsrOzJUn33HOPnnjiCcXExMjtduunP/2pevToofz8fB08eFCLFi1SfHx85X1nzpypN954QykpKXI6nWrWrJkSExO1d+9euVwuDR06VIWFhfr973+v0NBQ9e3bVwMGDFBQUJAaNWqk/Px8L/3UuJU9/vjjOnLkiH71q1+pXr16sixLY8aM0eLFi5WRkaFFixYpICBAf/rTn3TmzBmNHDlSn3/+uQICAtSyZUv+3tmIs1feRrZv366VK1dqxowZ3h4Ft5G4uDglJiaqdevW3h7ltsVDNwBgOHb0AGA4dvQAYDhCDwCGI/QAYDhCD2Nt375dUVFRiouLU1xcnAYMGKClS5fe8HGSk5O1evVq7d+/v/JUET9k8+bNysvLq9Exs7KylJCQcMOzAP8KXkcPo3Xp0qXy5aRlZWXq1auX+vbtq6CgoBs+VkREhCIiIqq8fcmSJUpMTFRYWNi/PC9gB0KP20ZRUZF8fHwUHx+vZs2a6eLFi5o7d64SExN14sQJud1uDR8+XJ07d9Y//vEPzZ49W6GhoSovL1d4ePg170NIS0vTu+++K7fbre7du+uBBx7Q/v37NXbsWK1YsULvvfee1q9fL4fDod69e2vIkCE6cuSIxo8fr4CAAAUEBCg4ONjbvxLcJgg9jJadna24uDg5HA75+flp4sSJmj9/vvr06aOePXtqxYoVatiwod58802dP39esbGx2rBhg6ZPn660tDSFhITo+eefv+aYZ8+e1bx587Ru3To5nU4lJSWpU6dOioiIUGJionJycrRx40atWLFCDodD8fHxio6O1ttvv62XX35ZDz/8sObOnaujR4966beC2w2hh9G++9DNt+bPn69WrVpJkg4ePKjPP/9cu3btkiS5XC6dOXNGDRo0UMOGDSVJkZGR19z/5MmTatu2rfz9/SXpe+dcP3jwoE6fPl15WooLFy4oJydHhw4dUvv27SVdPbkXoUdt4clY3Ja+PXFWeHi4nnzySS1dulTz5s1Tr169FBQUpEuXLuncuXOSrp6F8btatGiho0ePVp5N8eWXX1ZeXp4cDocsy1J4eLjatGmjJUuWaOnSpXr66afVrl07hYeHa+fOnZJUeZIvoDawo8dtbdCgQZowYYJiY2NVVFSkwYMHy+l0aurUqXr22WcVHBwsX99r/5mEhoZq6NChio2NlcPhULdu3RQWFqbIyEiNGTNGqampioqKUkxMjMrKytS+fXuFhYXptdde04gRI7RgwQKFhoaqbt26XvqpcbvhFAgAYDgeugEAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADDc/wd28yWV6BlNjQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "confusion_matrix = pd.crosstab(y_test, pred, rownames=['Actual'], colnames=['Predicted'])\n",
    "plt.figure(figsize = (6, 6))\n",
    "sns.heatmap(confusion_matrix, annot = True, cmap = 'Paired', cbar = False, fmt=\"d\", xticklabels=['Not Spam', 'Spam'], yticklabels=['Not Spam', 'Spam']);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5d3e988",
   "metadata": {},
   "source": [
    "## **Support Vector Machine (Linear)** <br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "fa99e0f4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 86.8 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "lsvc = LinearSVC()\n",
    "lsvc.fit(dtv, y_train)\n",
    "pred = lsvc.predict(test_dtv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "f26709ff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  96.42857142857143\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy: ', accuracy_score(y_test, pred) * 100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f112f5bf",
   "metadata": {},
   "source": [
    "**Classification Report**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "6d03481c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       184\n",
      "           1       0.69      0.75      0.72        12\n",
      "\n",
      "    accuracy                           0.96       196\n",
      "   macro avg       0.84      0.86      0.85       196\n",
      "weighted avg       0.97      0.96      0.96       196\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88a30ea0",
   "metadata": {},
   "source": [
    "**Confusion Matrix**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "54f18111",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAFxCAYAAAB9Z2NTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAZ6ElEQVR4nO3de1jUdf738dcgjOABkCzU20PgIWlNf7SasrKWqa1Z/nS7TMMFY2vNXdu6UFPRbKPMxFXSunfV1TwfsgVFTV27TS2sFnW1fh7zgAc8JOAJA0EY53v/4RXX+isEtvky+uH5uC6va5hh5vOGS599mpnvdxyWZVkCABjLx9sDAADsRegBwHCEHgAMR+gBwHCEHgAMR+gBwHC+3h7gf1s7OMLbIwA/qnD0Jm+PAJQrJrJpubexowcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoQcAwxF6ADAcoTdEcMv2+sWERZKkwBZtFf3GCnV9fan+64W3JIdDktS8+9Pq9laqot9YodDIR7w3LCCpIP+S3hn+jPLOZHt7FOP5ensA/HStnnxeTaP/W65rRZKk+556UYfTZyr36ww9+OKfFRr5sC5n7VP4r2KVMWGAfPxqK/r1Zcrb+4XcrlIvT4+a6LrLpXXvT5ev0+ntUWoEdvQGKMzJ1s4ZL5d9nX/ioJx1gyRJvv515Xa5FNzyAV08vFtuV6lcRQUqzMlWYPP7vDUyarj/t3S2Ovbsq/oNGnp7lBrBltBPnz5dXbt2VXR0dNkf2OfbnZtu2pkXnjuhds+OV/dp61U76C5dOLhDfgH1VHq1oOx7XMWF8q1T3xvjoob76tONqhMYrFYdOnl7lBrDlqduPvvsM23dulVO/rfMK9oNGa8v3ojTd2eO6t5eg/Wz34xV7p7P5RtQt+x7fP3rqrTwihenRE311acb5XA4dGzvbp07eVTpM5MVM/ot1Q8O8fZoxrIl9BEREbp27Rqh95KSwnyVFt3YvRdfylVIm0hdztqriIEJ8vFzysfXqXr/J1zfnT7i5UlREz2XNKPs8oI3RurJ3yUQeZvZEvrWrVsrOjpaDRs2lGVZcjgc2rx5sx1L4Uf8z9zX1PGlFLnd12W5SvX13Nd0Lf+8jn28VF3/tFQOHx998+EMuUtLvD0qgGrgsCzL8vSDDhgwQLNnz1ZgYGDZdZXd3a8dHOHpcQCPKBy9ydsjAOWKiWxa7m227OibNGmigIAAnroBgNuALaE/d+6cevXqpWbNmkmSHA6HVqxYYcdSAIAK2BL66dOn2/GwAID/gC2hd7lc2rhxo0pLb7y3Ozc3V2+++aYdSwEAKmDLAVNjx46VJO3evVunT5/W5cuX7VgGAFAJtoTe399fw4YNU2hoqJKTk3X+/Hk7lgEAVIItobcsS3l5eSosLNTVq1eVn59vxzIAgEqwJfR//OMftWnTJvXr1089evRQt27d7FgGAFAJtrwY26lTJ7Vs2VKnTp3SP/7xDwUHB9uxDACgEmwJ/bJly7Ro0SK1bt1aR48e1fDhw9WvXz87lgIAVMCW0Kempuqjjz5S7dq1VVRUpNjYWEIPAF5iy3P0d911l2rVqiXpxjtweOoGALzHlh29ZVnq37+/IiMjdeDAAblcLo0aNUqSlJKSYseSAIBy2BL63//+92WX+/bta8cSAIBK8njoP/nkE/Xs2VPfffedZs6cKafTqWHDhqlOnTqeXgoAUAkefY5+2rRpWrNmja5fv66JEyfq6tWratCggZKSkjy5DACgCjy6o9+/f78WLFggl8ulTz/9VJ999pkCAgIUExPjyWUAAFXg0R399++02bNnj9q0aaOAgABJKjuLJQCg+nl0R1+rVi19/vnnSk9P12OPPSZJ+vLLL2/6SEEAQPXy6I7+1VdfVVpamho1aqSYmBht27ZNycnJmjBhgieXAQBUgS0fDv5T8OHguF3x4eC4nd3qw8FtOTIWAHD7IPQAYDhbQp+amnrT14sXL7ZjGQBAJXj0XTfr1q3Tli1btH37dmVmZkqSrl+/riNHjmjIkCGeXAoAUEkeDf0vf/lL3X333bp8+bIGDRokSfLx8VGzZs08uQwAoAo8+tRNUFCQOnfurPnz56uoqEh79uzR5cuXFRoa6sllAABVYMtz9CkpKUpLS5Ovr69Wr16t5ORkO5YBAFSCLacp3rlzp1asWCFJevbZZzVw4EA7lgEAVIItO3qXyyW32y3pxoeQOBwOO5YBAFSCLTv6Pn36KCYmRh06dNCePXvUp08fO5YBAFSCLaF/7rnnFB0drWPHjmnAgAFq06aNHcsAACrBo6FfvXr1D647cOCADhw4oP79+3tyKQBAJXk09FlZWTd9bVmWVq1aJX9/f0IPAF7i0dCPGjWq7PLJkyeVmJioRx55ROPHj/fkMgCAKrDlOfply5Zp0aJFGjdunLp3727HEgCASvJo6HNycjRu3DgFBQUpNTVVQUFBnnx4AMB/wKOhf/LJJ+Xn56cuXbrozTffvOm2lJQUTy4FAKgkj4b+r3/9qycfDgDgAR4N/UMPPeTJhwMAeACfMAUAhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhvMt74bPP/+83DtFR0fbMgwAwPPKDf369evLvROhB4A7R7mhnzx58o9en5uba9swAADPKzf033vvvfe0fPlylZaWqri4WPfee+8td/sAgNtLhS/GZmRkKCMjQ3379tWGDRsUGhpaHXMBADykwtAHBwfL6XSqsLBQLVq0UFFRUXXMBQDwkApD36hRI6WlpSkgIEApKSkqKCiojrkAAB7isCzLutU3uN1uffvttwoKClJ6erqioqLUqlUr2wZaOzjCtscGforC0Zu8PQJQrpjIpuXeVuGLsWvXri27XL9+fe3bt8/W0AMAPKvC0GdlZUmSLMvSwYMHFRwcrP79+9s9FwDAQyoM/ahRo8ouW5alYcOG2ToQAMCzKgx9SUlJ2eW8vDydPn3a1oEAAJ5VYeh79+4th8Mhy7Lk7++v3/3ud7YOxAteAOBZFYZ+xowZat++fdnXO3bssHUgAIBnlRv6f/3rXzp69KgWLlyo3/72t5JuvNVy2bJlWrduXbUNCAD4acoNfWBgoM6fP6+SkhLl5eVJkhwOh0aPHl1twwEAfroKD5jKycnRxYsXFRERoU8++UQPP/yw/Pz8bBvog694sRcAqupWB0xVeAqESZMm6euvv5YkHT9+XImJiR4bDABgvwpDn5OTo5iYGEnS0KFDOR89ANxhKvWZscePH5cknTx5Um6329aBAACeVeHbK8ePH6+EhARduHBB/v7++vWvf10dcwEAPKTCHX2HDh00ceJE/eIXv1BRUZEuXLhQHXMBADyk3B19SUmJ1q9fr2XLlsnpdKqgoECbN2+Wv79/dc4HAPiJyt3RP/roozp06JCmTZum5cuX65577iHyAHAHKndHP2TIEK1bt05nzpzRgAEDVMHb7QEAt6kKD5jasWOHUlNTlZGRoQEDBqhfv35q06aNbQNxwBQAVN2tDpiqMPTfu3LlitasWaOVK1dq9erVnprtBwg9AFSdR0JfXQg9AFTdTzoFAgDgzkboAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwvt4eAPZwu69r7Zx3dOHsKTl8fNT/92MU0qiJt8cC5Cot0epZU3Up91vVDqijJ557WXc1burtsYzGjt5Qh3b9U5L0/JvvqfvT8fp4ySwvTwTcsGvzBjn9AzT0rb+oz29f0oYF/9fbIxmPHb2hIjpFq82DUZKk/PM5qhvUwMsTATfknTmp1v/1kCSpYZNmyjuT7eWJzGdb6E+dOqWtW7fq2rVrZdcNHTrUruXwI2rVqqX0mck6uPMLDUx43dvjAJKkRi1a6vDuf6ptp646ffSgvrt4Xm73dfn41PL2aMay7amb4cOHKz8/X06ns+wPqt+vhyfqpemL9NHcFJUUF3l7HECR3R9X7YC6WvjmKB3e9U81Dm9N5G1m246+cePGeumll+x6eFTgfzI26crFPP2y/2D5OWvL4fCRg39MuA2czfpGzdu2U+9nh+tM1iFdzDnr7ZGM57Asy7LjgT/44AOdOXNGrVq1Kruuf//+Fd/vq9N2jFPjlBQXafXsqSq4fFHu6y5F94tR245dvT0WoMIr+Up77y2VXiuSf516+u9hrygwpKG3x7rjxUSW/84l20IfFxen8PBwBQYG3ljI4dDIkSMrvB+hB4Cqu1XobXvqxul06o033rDr4QEAlWRb6Js0aaK//e1vuv/+++VwOCRJ0dHRdi0HACiHbaF3uVw6ceKETpw4UXYdoQeA6mdb6CdPnnzT17m5uXYtBQC4BdtC/95772n58uUqLS1VcXGx7r33Xq1fv96u5QAA5bDtgKmMjAxlZGSob9++2rBhg0JDQ+1aCgBwC7aFPjg4WE6nU4WFhWrRooWKijgqEwC8wbbQN2rUSGlpaQoICNC0adNUUFBg11IAgFuw7YApt9utc+fOKTAwUOnp6YqKirrpKNnycMAUAFSdVw6YunTpkubPn68TJ06odevWuvvuu+1aCgBwC7Y9dZOQkKDw8HC98soratq0qcaMGWPXUgCAW7D1g0cGDx4sSWrbtq02btxo51IAgHLYtqMPDw/XmjVrlJOToy1btig4OFjHjx/X8ePH7VoSAPAjbD17pSRduXJFtWrVUt26dW8s6HBo8eLF5d6PF2MBoOpu9WKsx3f0+/fvV//+/TVv3jzFxcUpLy9PhYWFio+P15IlS24ZeQCA53k89NOnT1dycrKcTqdmzJihuXPnauXKlZo7d66nlwIAVILHX4y1LEtt27ZVTk6OioqK9LOf/UySyk5VDACoXh7f0bvdbknStm3bFBUVJUkqKSnR1atXPb0UAKASPL6jj4qK0jPPPKNz585p1qxZys7OVlJSkvr06ePppQAAlWDLu26ysrIUEhKiBg0aKDs7W4cOHVKvXr0qdV/edQMAVVftp0Bo2bJl2eXmzZurefPmdiwDAKgE2w6YAgDcHgg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4RyWZVneHgIAYB929ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIbz9fYAuGH79u168cUX9dFHH6lx48aSpGnTpik8PFxPPfXUj97n8uXL2rZtm/r27XvT9SdPntSkSZN0/fp1uVwutWvXTqNGjZKPD/9dh73mzJmjL7/8Uj4+PnI4HBoxYoTatWvn7bFqPP7l30b8/Pw0btw4VfbQhkOHDmnLli0/uP6dd95RbGys5s2bp4ULF+rEiRPavHmzp8cFbnL06FFt2bJFCxYs0Pz58/XKK69o/Pjx3h4LYkd/W+nSpYvcbreWLVum2NjYm26bP3++1q9fL19fX3Xs2FGjR4/W7Nmz9c033+jDDz/UoEGDyr63SZMmSk9PV926ddW+fXvNmDFDvr6+2r59u2bPni0fHx/l5eVp0KBB+s1vfqMdO3boL3/5iySpuLhYU6ZMkZ+fn0aMGKHGjRvr9OnTeuKJJ3TkyBEdOHBAjzzyiEaOHFmtvxvc/kJCQnT27FmlpaWpW7duioiIUFpamuLi4hQWFqbjx4/LsixNnz5dISEh+tOf/qRz587p0qVL6tatmxISEpSYmChfX1+dPXtWJSUl6tOnj7Zu3apvv/1WM2fOVPPmzb39Y96ZLNwWMjMzrYSEBOvixYtWjx49rOPHj1tTp061Vq5caX3zzTfWgAEDrJKSEsvtdlsvvviitWXLlrL7/G/Xrl2zFixYYA0ePNjq2LGjNWrUKCs/P9/KzMy0Hn/8cevatWtWUVGR1bNnT+v8+fPW0qVLrXPnzlmWZVmzZs2yZs6caZ06dcrq3LmzdeXKFSs3N9d64IEHrEuXLlnFxcVWVFRUdf96cIfYt2+flZiYaD388MPWr371K2vjxo1WbGyslZ6eblmWZS1dutSaOHGiderUKevvf/+7ZVmWVVxcbD300EOWZVnW2LFjrZkzZ1qWZVmvvfaaNWXKFMuyLOvdd9+1FixYUO0/jynY0d9mGjRooPHjxysxMVEPPvigJOnYsWPq0KGD/Pz8JEkdO3bUkSNH1KFDhx99jMzMTMXHxys+Pl6FhYWaMmWKZs6cqe7duysyMlJOp1OS1Lp1a2VnZys0NFSTJk1SnTp1lJOTU7Zus2bNVL9+fTmdTjVs2FDBwcGSJIfDYfNvAXeikydPql69epo8ebIkae/evXrhhRfUsGFDdenSRZL04IMPasuWLQoODtbevXuVmZmpevXqqaSkpOxx7r//fklSYGCgwsPDyy7/+/eganiO/jb06KOPKiwsTOnp6ZKk8PBw7dmzRy6XS5ZlaefOnQoLC5OPj4/cbvcP7j916lR98cUXkqS6desqLCysLO4HDx7U9evXVVRUpKNHj6pFixaaMGGC3n77bSUnJ+uee+4pe42AoKMqDh06pKSkJF27dk2SFBYWpvr166tWrVrat2+fJGn37t1q1aqVVq1apfr16yslJUXPPfeciouL+XtnI3b0t6lXX31VmZmZkqT77rtPjz/+uGJiYuR2u/Xzn/9cPXv2VG5urg4fPqyFCxcqPj6+7L4zZszQW2+9pZSUFDmdTjVt2lRJSUnav3+/XC6Xhg4dqsuXL+sPf/iDQkJC1K9fPw0cOFCBgYFq2LChcnNzvfRT40722GOPKSsrS08//bTq1Kkjy7I0ZswYLVq0SOnp6Vq4cKECAgL05z//WefPn9fIkSO1a9cuBQQEqEWLFvy9sxFnr6xBtm/frhUrVmj69OneHgU1SFxcnJKSktSyZUtvj1Jj8dQNABiOHT0AGI4dPQAYjtADgOEIPQAYjtDDWNu3b1dUVJTi4uIUFxengQMHasmSJVV+nGnTpmnVqlU6ePBg2akifsymTZuUk5NTqcfMyMhQYmJilWcB/hO8jx5G69KlS9nbSUtKStS7d2/169dPgYGBVX6siIgIRURElHv74sWLlZSUpNDQ0P94XsAOhB41RkFBgXx8fBQfH6+mTZvqypUrmjNnjpKSknTy5Em53W4lJCSoc+fO+vjjjzVr1iyFhISotLRU4eHhNx2HkJqaqg8++EBut1s9evTQAw88oIMHD2rs2LFavny5PvzwQ61bt04Oh0N9+vTRkCFDlJWVpfHjxysgIEABAQEKCgry9q8ENQShh9EyMzMVFxcnh8MhPz8/vfbaa3r//ffVt29f9erVS8uXL1eDBg309ttv69KlS4qNjdX69es1depUpaamKjg4WC+88MJNj3nhwgXNnTtXa9euldPpVHJysjp16qSIiAglJSUpOztbGzZs0PLly+VwOBQfH6/o6Gi9++67evnll9W1a1fNmTNHx44d89JvBTUNoYfR/v2pm++9//77CgsLkyQdPnxYu3bt0p49eyRJLpdL58+fV7169dSgQQNJUmRk5E33P3XqlFq3bi1/f39J+sE51w8fPqyzZ8+WnZYiPz9f2dnZOnLkiNq3by/pxsm9CD2qCy/Gokb6/sRZ4eHheuKJJ7RkyRLNnTtXvXv3VmBgoL777jtdvHhR0o2zMP675s2b69ixY2VnU3z55ZeVk5Mjh8Mhy7IUHh6uVq1aafHixVqyZImeeuoptWnTRuHh4frqq68kqewkX0B1YEePGu2ZZ57RhAkTFBsbq4KCAg0ePFhOp1OTJ0/W888/r6CgIPn63vzPJCQkREOHDlVsbKwcDoe6d++u0NBQRUZGasyYMZo/f76ioqIUExOjkpIStW/fXqGhoXr99dc1YsQIzZs3TyEhIapdu7aXfmrUNJwCAQAMx1M3AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8Ahvv/d7U2xhfYmF0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "confusion_matrix = pd.crosstab(y_test, pred, rownames=['Actual'], colnames=['Predicted'])\n",
    "plt.figure(figsize = (6, 6))\n",
    "sns.heatmap(confusion_matrix, annot = True, cmap = 'Paired', cbar = False, fmt=\"d\", xticklabels=['Not Spam', 'Spam'], yticklabels=['Not Spam', 'Spam']);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c395fbf7",
   "metadata": {},
   "source": [
    "## **Decision Tree Classifier** <br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "ef0e06e1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 912 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "dtc = DecisionTreeClassifier()\n",
    "dtc.fit(dtv, y_train)\n",
    "pred = dtc.predict(test_dtv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "86e1648e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  95.91836734693877\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy: ', accuracy_score(y_test, pred) * 100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f91f8976",
   "metadata": {},
   "source": [
    "**Classification Report**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "ecb89f02",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.97      0.98       184\n",
      "           1       0.64      0.75      0.69        12\n",
      "\n",
      "    accuracy                           0.96       196\n",
      "   macro avg       0.81      0.86      0.84       196\n",
      "weighted avg       0.96      0.96      0.96       196\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09c5b503",
   "metadata": {},
   "source": [
    "**Confusion Matrix**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "f8eba58c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAFxCAYAAAB9Z2NTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAaEElEQVR4nO3deXTU9b3/8deEZEhYsokEKIsJi8Qi3FgUonFBwCLKhfqjYGiCUYu0Wv0BEQiINRWRUIigbYGC7BKxBAIIFC8CNagNULRFFlkDCSBZ2AlJJsl87x8cc8rVkKTONwMfno9zOCeZYb6fd3Lk6ed8Z+Y7DsuyLAEAjOXj7QEAAPYi9ABgOEIPAIYj9ABgOEIPAIYj9ABgOF9vD/B/rRkS6e0RgO9VNHqjt0cAqhQb1bLK+9jRA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGM7X2wPAM4LbdtYdsYn6/I2n9JMXU1U/qIkkqcGtP9LZQ//Szj8kql2/X+pH0X1VXlykQ2vnKe/Lv3l3aNy0ZicNV/2AhpKkkKbNNODXY7w8kdkIvQHaPf6sWsb8t8pLiyVJO/+QKEnyaxioe19ZqN1LUtS4VXv96N7HtPW3gyVJMcnvq3BPlipcJV6bGzenMpdLkvT0a295eZKbB6duDFCUl6MdM176zu23/7/fKPt/lqr0XIEat2ir0/u2y13mkrvMpaJTxxTY+nYvTIubXd6xwyorLdHiSWO0cGKicg/u9fZIxrMl9NOnT9d9992nmJiYyj+wzzc7NspdXnbVbc7AUDXpFK2cTzIkSRdyD+iWjl1Vz7+B/BoFK7TDf6le/QBvjIubnF/9+rr38UGKHz9Fjz87Qiv/8KYqKiq8PZbRbDl188knn2jLli1yOp12HB410OKen+rEZ2slyy1JunTyiLL/J03dx85R0akcnT20S66LZ708JW5GtzRvqdBmP5LD4VCTFq0U0DhQl86eVlCTpt4ezVi27OgjIyNVWlpqx6FRQ7d2ilb+v7ZWfu9sHCJn4xB99rs47V78pgJuaaYLuQe9OCFuVl9u2aCPlsyWJF04U6jS4stqFHKLl6cymy07+vbt2ysmJkZNmjSRZVlyOBzatGmTHUuhCg1bhKsoP7fye9fFs2rQtKXun/gXWeVl2pM2rXK3D9SlqIcf1aqZv9e81/6/HJL6Dx+tevXqeXssozksy7I8fdCBAwdq9uzZCgwMrLytpqdx1gyJ9PQ4gEcUjd7o7RGAKsVGtazyPlt29C1atFBAQADn6AHgOmBL6E+dOqXevXurVatWkiSHw6Fly5bZsRQAoBq2hH769Ol2HBYA8B+wJfTl5eXasGGDysquvLY7Pz9fr7/+uh1LAQCqYcvLK8eOHStJ+uKLL3T8+HGdO3fOjmUAADVgS+j9/f01fPhwhYWFKSUlRYWFhXYsAwCoAVtCb1mWCgoKVFRUpMuXL+v8+fN2LAMAqAFbQv+b3/xGGzduVP/+/dWzZ0898MADdiwDAKgBW56Mvfvuu9W2bVvl5ubqr3/9q4KDg+1YBgBQA7aEfunSpVq0aJHat2+vQ4cO6fnnn1f//v3tWAoAUA1bQr98+XJ9+OGHql+/voqLixUXF0foAcBLbDlHf8stt1RepMjf359TNwDgRbbs6C3L0oABAxQVFaW9e/eqvLxciYlXPt4uNTXVjiUBAFWwJfS/+tWvKr/u16+fHUsAAGrI46H/+OOP1atXL128eFEzZ86U0+nU8OHD1aBBA08vBQCoAY+eo582bZpWr16tiooKTZw4UZcvX1ZISIiSk5M9uQwAoBY8uqPfs2ePFixYoPLycv3tb3/TJ598ooCAAMXGxnpyGQBALXh0R//tK2127dqlDh06KCAgQJIqr2IJAKh7Ht3R16tXT59++qkyMjL0yCOPSJI+//zzqz5SEABQtzy6o3/llVeUnp6uZs2aKTY2Vlu3blVKSoomTJjgyWUAALVgy4eD/xB8ODiuV3w4OK5n1/pwcFveGQsAuH4QegAwnC2hX758+VXfL1682I5lAAA14NFX3axdu1abN2/Wtm3blJWVJUmqqKjQwYMHNXToUE8uBQCoIY+G/v7779ett96qc+fOafDgwZIkHx8ftWrVypPLAABqwaOnboKCgtStWzfNnz9fxcXF2rVrl86dO6ewsDBPLgMAqAVbztGnpqYqPT1dvr6+WrVqlVJSUuxYBgBQA7ZcpnjHjh1atmyZJOmpp57SoEGD7FgGAFADtuzoy8vL5Xa7JV35EBKHw2HHMgCAGrBlR9+3b1/FxsaqS5cu2rVrl/r27WvHMgCAGrAl9M8884xiYmJ05MgRDRw4UB06dLBjGQBADXg09KtWrfrObXv37tXevXs1YMAATy4FAKghj4b+8OHDV31vWZZWrlwpf39/Qg8AXuLR0CcmJlZ+fezYMSUlJemhhx7S+PHjPbkMAKAWbDlHv3TpUi1atEjjxo1Tjx497FgCAFBDHg19Xl6exo0bp6CgIC1fvlxBQUGePDwA4D/g0dA//vjj8vPzU/fu3fX6669fdV9qaqonlwIA1JBHQ/+nP/3Jk4cDAHiAR0N/zz33ePJwAAAP4BOmAMBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwhB4ADEfoAcBwvlXd8emnn1b5oJiYGFuGAQB4XpWhX7duXZUPIvQAcOOoMvSTJ0/+3tvz8/NtGwYA4HlVhv5b77zzjtLS0lRWVqaSkhLddttt19ztAwCuL9U+GZuZmanMzEz169dP69evV1hYWF3MBQDwkGpDHxwcLKfTqaKiIrVp00bFxcV1MRcAwEOqDX2zZs2Unp6ugIAApaam6tKlS3UxFwDAQxyWZVnX+gtut1vffPONgoKClJGRoejoaLVr1862gdYMibTt2MAPUTR6o7dHAKoUG9WyyvuqfTJ2zZo1lV83btxYu3fvtjX0AADPqjb0hw8fliRZlqV9+/YpODhYAwYMsHsuAICHVBv6xMTEyq8ty9Lw4cNtHQgA4FnVht7lclV+XVBQoOPHj9s6EADAs6oNfZ8+feRwOGRZlvz9/fXLX/7S1oF4wgsAPKva0M+YMUOdO3eu/H779u22DgQA8KwqQ/+Pf/xDhw4d0sKFC/X0009LuvJSy6VLl2rt2rV1NiAA4IepMvSBgYEqLCyUy+VSQUGBJMnhcGj06NF1NhwA4Ier9g1TeXl5OnPmjCIjI/Xxxx/rwQcflJ+fn20Dvf8lT/YCQG1d6w1T1V4CYdKkSfrnP/8pScrOzlZSUpLHBgMA2K/a0Ofl5Sk2NlaSNGzYMK5HDwA3mBp9Zmx2drYk6dixY3K73bYOBADwrGpfXjl+/HiNGDFCp0+flr+/v372s5/VxVwAAA+pdkffpUsXTZw4Uffee6+Ki4t1+vTpupgLAOAhVe7oXS6X1q1bp6VLl8rpdOrSpUvatGmT/P3963I+AMAPVOWO/uGHH9b+/fs1bdo0paWlqWnTpkQeAG5AVe7ohw4dqrVr1+rEiRMaOHCgqnm5PQDgOlXtG6a2b9+u5cuXKzMzUwMHDlT//v3VoUMH2wbiDVMAUHvXesNUtaH/1oULF7R69WqtWLFCq1at8tRs30HoAaD2PBL6ukLoAaD2ftAlEAAANzZCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCGI/QAYDhCDwCG8/X2ALCH212hNXPe0umTuXL4+GjAr8YotFkLb48FqLzMpVWzpups/jeqH9BAjz3zkm5p3tLbYxmNHb2h9u/8uyTp2dffUY+fJ+ijJbO8PBFwxc5N6+X0D9CwN/6ovk+/qPUL/uDtkYzHjt5QkXfHqMNd0ZKk84V5ahgU4uWJgCsKThxT+/+6R5LUpEUrFZzI8fJE5rMt9Lm5udqyZYtKS0srbxs2bJhdy+F71KtXTxkzU7Rvx2caNOI1b48DSJKatWmrA1/8XR3vvk/HD+3TxTOFcrsr5ONTz9ujGcu2UzfPP/+8zp8/L6fTWfkHde9nzyfpxemL9OHcVLlKir09DqCoHo+qfkBDLXw9UQd2/l3NI9oTeZvZtqNv3ry5XnzxRbsOj2r8K3OjLpwp0P0DhsjPWV8Oh48c/GPCdeDk4a/VumMn9XnqeZ04vF9n8k56eyTjOSzLsuw48Pvvv68TJ06oXbt2lbcNGDCg+sd9edyOcW46rpJirZo9VZfOnZG7olwx/WPVset93h4LUNGF80p/5w2VlRbLv0Ej/ffwlxUY2sTbY93wYqOqfuWSbaGPj49XRESEAgMDryzkcGjUqFHVPo7QA0DtXSv0tp26cTqd+t3vfmfX4QEANWRb6Fu0aKE///nPuuOOO+RwOCRJMTExdi0HAKiCbaEvLy/X0aNHdfTo0crbCD0A1D3bQj958uSrvs/Pz7drKQDANdgW+nfeeUdpaWkqKytTSUmJbrvtNq1bt86u5QAAVbDtDVOZmZnKzMxUv379tH79eoWFhdm1FADgGmwLfXBwsJxOp4qKitSmTRsVF/OuTADwBttC36xZM6WnpysgIEDTpk3TpUuX7FoKAHANtr1hyu1269SpUwoMDFRGRoaio6OvepdsVXjDFADUnlfeMHX27FnNnz9fR48eVfv27XXrrbfatRQA4BpsO3UzYsQIRURE6OWXX1bLli01ZswYu5YCAFyDrR88MmTIEElSx44dtWHDBjuXAgBUwbYdfUREhFavXq28vDxt3rxZwcHBys7OVnZ2tl1LAgC+h61Xr5SkCxcuqF69emrYsOGVBR0OLV68uMrH8WQsANTetZ6M9fiOfs+ePRowYIDmzZun+Ph4FRQUqKioSAkJCVqyZMk1Iw8A8DyPh3769OlKSUmR0+nUjBkzNHfuXK1YsUJz58719FIAgBrw+JOxlmWpY8eOysvLU3FxsX784x9LUuWligEAdcvjO3q32y1J2rp1q6KjoyVJLpdLly9f9vRSAIAa8PiOPjo6Wk8++aROnTqlWbNmKScnR8nJyerbt6+nlwIA1IAtr7o5fPiwQkNDFRISopycHO3fv1+9e/eu0WN51Q0A1F6dXwKhbdu2lV+3bt1arVu3tmMZAEAN2PaGKQDA9YHQA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGM5hWZbl7SEAAPZhRw8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4X28PgCu2bdumF154QR9++KGaN28uSZo2bZoiIiL0xBNPfO9jzp07p61bt6pfv35X3X7s2DFNmjRJFRUVKi8vV6dOnZSYmCgfH/6/DnvNmTNHn3/+uXx8fORwODRy5Eh16tTJ22Pd9PiXfx3x8/PTuHHjVNO3Nuzfv1+bN2/+zu1vvfWW4uLiNG/ePC1cuFBHjx7Vpk2bPD0ucJVDhw5p8+bNWrBggebPn6+XX35Z48eP9/ZYEDv660r37t3ldru1dOlSxcXFXXXf/PnztW7dOvn6+qpr164aPXq0Zs+era+//loffPCBBg8eXPl3W7RooYyMDDVs2FCdO3fWjBkz5Ovrq23btmn27Nny8fFRQUGBBg8erF/84hfavn27/vjHP0qSSkpKNGXKFPn5+WnkyJFq3ry5jh8/rscee0wHDx7U3r179dBDD2nUqFF1+rvB9S80NFQnT55Uenq6HnjgAUVGRio9PV3x8fEKDw9Xdna2LMvS9OnTFRoaqt/+9rc6deqUzp49qwceeEAjRoxQUlKSfH19dfLkSblcLvXt21dbtmzRN998o5kzZ6p169be/jFvTBauC1lZWdaIESOsM2fOWD179rSys7OtqVOnWitWrLC+/vpra+DAgZbL5bLcbrf1wgsvWJs3b658zP9VWlpqLViwwBoyZIjVtWtXKzEx0Tp//ryVlZVlPfroo1ZpaalVXFxs9erVyyosLLTee+8969SpU5ZlWdasWbOsmTNnWrm5uVa3bt2sCxcuWPn5+dadd95pnT171iopKbGio6Pr+teDG8Tu3butpKQk68EHH7R++tOfWhs2bLDi4uKsjIwMy7Is67333rMmTpxo5ebmWn/5y18sy7KskpIS65577rEsy7LGjh1rzZw507Isy3r11VetKVOmWJZlWW+//ba1YMGCOv95TMGO/joTEhKi8ePHKykpSXfddZck6ciRI+rSpYv8/PwkSV27dtXBgwfVpUuX7z1GVlaWEhISlJCQoKKiIk2ZMkUzZ85Ujx49FBUVJafTKUlq3769cnJyFBYWpkmTJqlBgwbKy8urXLdVq1Zq3LixnE6nmjRpouDgYEmSw+Gw+beAG9GxY8fUqFEjTZ48WZL01Vdf6bnnnlOTJk3UvXt3SdJdd92lzZs3Kzg4WF999ZWysrLUqFEjuVyuyuPccccdkqTAwEBFRERUfv3vfwe1wzn669DDDz+s8PBwZWRkSJIiIiK0a9culZeXy7Is7dixQ+Hh4fLx8ZHb7f7O46dOnarPPvtMktSwYUOFh4dXxn3fvn2qqKhQcXGxDh06pDZt2mjChAl68803lZKSoqZNm1Y+R0DQURv79+9XcnKySktLJUnh4eFq3Lix6tWrp927d0uSvvjiC7Vr104rV65U48aNlZqaqmeeeUYlJSX8d2cjdvTXqVdeeUVZWVmSpNtvv12PPvqoYmNj5Xa79ZOf/ES9evVSfn6+Dhw4oIULFyohIaHysTNmzNAbb7yh1NRUOZ1OtWzZUsnJydqzZ4/Ky8s1bNgwnTt3Tr/+9a8VGhqq/v37a9CgQQoMDFSTJk2Un5/vpZ8aN7JHHnlEhw8f1s9//nM1aNBAlmVpzJgxWrRokTIyMrRw4UIFBATo97//vQoLCzVq1Cjt3LlTAQEBatOmDf/d2YirV95Etm3bpmXLlmn69OneHgU3kfj4eCUnJ6tt27beHuWmxakbADAcO3oAMBw7egAwHKEHAMMRegAwHKGHsbZt26bo6GjFx8crPj5egwYN0pIlS2p9nGnTpmnlypXat29f5aUivs/GjRuVl5dXo2NmZmYqKSmp1rMA/wleRw+jde/evfLlpC6XS3369FH//v0VGBhY62NFRkYqMjKyyvsXL16s5ORkhYWF/cfzAnYg9LhpXLp0ST4+PkpISFDLli114cIFzZkzR8nJyTp27JjcbrdGjBihbt266aOPPtKsWbMUGhqqsrIyRUREXPU+hOXLl+v999+X2+1Wz549deedd2rfvn0aO3as0tLS9MEHH2jt2rVyOBzq27evhg4dqsOHD2v8+PEKCAhQQECAgoKCvP0rwU2C0MNoWVlZio+Pl8PhkJ+fn1599VW9++676tevn3r37q20tDSFhITozTff1NmzZxUXF6d169Zp6tSpWr58uYKDg/Xcc89ddczTp09r7ty5WrNmjZxOp1JSUnT33XcrMjJSycnJysnJ0fr165WWliaHw6GEhATFxMTo7bff1ksvvaT77rtPc+bM0ZEjR7z0W8HNhtDDaP9+6uZb7777rsLDwyVJBw4c0M6dO7Vr1y5JUnl5uQoLC9WoUSOFhIRIkqKioq56fG5urtq3by9/f39J+s411w8cOKCTJ09WXpbi/PnzysnJ0cGDB9W5c2dJVy7uRehRV3gyFjelby+cFRERoccee0xLlizR3Llz1adPHwUGBurixYs6c+aMpCtXYfx3rVu31pEjRyqvpvjSSy8pLy9PDodDlmUpIiJC7dq10+LFi7VkyRI98cQT6tChgyIiIvTll19KUuVFvoC6wI4eN7Unn3xSEyZMUFxcnC5duqQhQ4bI6XRq8uTJevbZZxUUFCRf36v/mYSGhmrYsGGKi4uTw+FQjx49FBYWpqioKI0ZM0bz589XdHS0YmNj5XK51LlzZ4WFhem1117TyJEjNW/ePIWGhqp+/fpe+qlxs+ESCABgOE7dAIDhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGO5/AcNCOcsPO400AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "confusion_matrix = pd.crosstab(y_test, pred, rownames=['Actual'], colnames=['Predicted'])\n",
    "plt.figure(figsize = (6, 6))\n",
    "sns.heatmap(confusion_matrix, annot = True, cmap = 'Paired', cbar = False, fmt=\"d\", xticklabels=['Not Spam', 'Spam'], yticklabels=['Not Spam', 'Spam']);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a66d9a7f",
   "metadata": {},
   "source": [
    "## **K Nearest Neighbours**<br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "03a1eb9d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 74.8 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "knn = KNeighborsClassifier()\n",
    "knn.fit(dtv, y_train)\n",
    "pred = knn.predict(test_dtv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "f27f2e76",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  96.42857142857143\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy: ', accuracy_score(y_test, pred) * 100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6443de8a",
   "metadata": {},
   "source": [
    "**Classification Report**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "b97399c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       184\n",
      "           1       0.73      0.67      0.70        12\n",
      "\n",
      "    accuracy                           0.96       196\n",
      "   macro avg       0.85      0.83      0.84       196\n",
      "weighted avg       0.96      0.96      0.96       196\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96c30ad6",
   "metadata": {},
   "source": [
    "**Confusion Matrix**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "43035f7d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAFxCAYAAAB9Z2NTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAZhElEQVR4nO3de1TVdb7/8ddG2IIXQLJQl5fEy0RTemw0ZWIqUxuzXFo/R8MBh6nMGZ08qKloOlGOiSOk9ZtBxwvekmxAUVPH+ZlYaB3U6XLMS17xlnHxghcEcbO/vz9cceIUAtP+svXj87FWa23Y7u/nDcueffruvb/bYVmWJQCAsXy8PQAAwF6EHgAMR+gBwHCEHgAMR+gBwHCEHgAM5+vtAf63dUPDvT0C8IOKx2/29ghAlaK6tKzyPnb0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0AGA4Qg8AhiP0hghu10k/n7JUkhTY5h5FvrZSD736jv7jxT9JDkfFn3M2bqLH3twkHz+nt0bFbc7tLteaebO06I+jlZoQp3N5p709kvEIvQHaP/W8/mP4NPn41Zck/eSZUTqYmaKPX4uWj59ToV0ekSTd2ekhRUxaqPqBd3hzXNzmDnz6X5Kk519/Wz1/Fat/Lp/r5YnMR+gNUJx/QrvmjK74+sKx/XI2DJIk+fo3lNvlun6H29Inbzyva8UXvDEmIEkK7xap/sPHSpIunMlXw6AmXp7IfL52HHT27NnKyMiQ4zunDLZv327HUpD0za7NCmjaouLr4rxjuv+3U9Xh6d/JdeWSzu7fKUkq3POJt0YEKqlXr54yUxK1f9fHGhz3qrfHMZ4tof/oo4+0detWOZ2cB/aG+4ZN1sevxejS14d1d5+h+umvJ+rLJdO8PRZQydMj49W76JwWThmlUUmpcvoHeHskY9ly6iY8PFxXr16149CogbLiC7pWclmSVHq+QH4NA708EfA//jt7s7atSZMk+Tnry+HwkcOnnpenMpstO/oOHTooMjJSTZs2lWVZcjgc2rJlix1L4Qf894Kp6vpSstzuclmua/piwVRvjwRUCH8wUmvmzVJqQpzc5S71/c1I+fF//7ZyWJZlefqggwYN0rx58xQY+D87yZqexlk3NNzT4wAeUTx+s7dHAKoU1aVllffZsqNv0aKFAgICOEcPADcBW0Kfl5enPn36qFWrVpIkh8OhlStX2rEUAKAatr28EgBwc7Al9C6XS5s2bdK1a9ckSQUFBXr99dftWAoAUA1bXl45ceJESdJnn32mU6dOqaioyI5lAAA1YEvo/f39NWLECIWGhioxMVFnzpyxYxkAQA3YEnrLslRYWKji4mJduXJFFy5wbRUA8BZbQv+HP/xBmzdv1oABA9SrVy89/PDDdiwDAKgBW56M7datm9q1a6eTJ0/qH//4h4KDg+1YBgBQA7aEfsWKFVq6dKk6dOigw4cPa+TIkRowYIAdSwEAqmFL6NPT0/X++++rfv36KikpUXR0NKEHAC+x5Rz9HXfcoXr1rl+Nzt/fn1M3AOBFtuzoLcvSwIED1aVLF+3bt08ul0vjxo2TJCUnJ9uxJACgCraE/ne/+13F7f79+9uxBACghjwe+g8++EC9e/fWpUuXlJKSIqfTqREjRqhBgwaeXgoAUAMePUeflJSktWvXqry8XNOmTdOVK1fUpEkTJSQkeHIZAEAteHRHv3fvXi1evFgul0sffvihPvroIwUEBCgqKsqTywAAasGjO/pvX2mze/dudezYUQEB1z/s99urWAIA6p5Hd/T16tXT9u3blZmZqccff1yS9Mknn1T6SEEAQN3y6I7+lVdeUUZGhpo1a6aoqCht27ZNiYmJmjJliieXAQDUgi0fDv5j8OHguFnx4eC4md3ow8FteWcsAODmQegBwHC2hD49Pb3S18uWLbNjGQBADXj0VTfr169XVlaWduzYoZycHElSeXm5Dh06pGHDhnlyKQBADXk09L/4xS905513qqioSEOGDJEk+fj4qFWrVp5cBgBQCx49dRMUFKTu3bsrNTVVJSUl2r17t4qKihQaGurJZQAAtWDLOfrk5GRlZGTI19dXa9asUWJioh3LAABqwJbLFO/atUsrV66UJP3mN7/R4MGD7VgGAFADtuzoXS6X3G63pOsfQuJwOOxYBgBQA7bs6Pv166eoqCh17txZu3fvVr9+/exYBgBQA7aE/rnnnlNkZKSOHj2qQYMGqWPHjnYsAwCoAY+Gfs2aNd/73r59+7Rv3z4NHDjQk0sBAGrIo6E/cuRIpa8ty9Lq1avl7+9P6AHASzwa+nHjxlXcPn78uOLj4/Xoo49q8uTJnlwGAFALtpyjX7FihZYuXapJkyapZ8+ediwBAKghj4Y+Pz9fkyZNUlBQkNLT0xUUFOTJwwMA/g0eDf1TTz0lPz8/9ejRQ6+//nql+5KTkz25FACghjwa+r/+9a+ePBwAwAM8GvoHH3zQk4cDAHgAnzAFAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIbzreqO7du3V/mgyMhIW4YBAHhelaHfsGFDlQ8i9ABw66gy9DNmzPjB7xcUFNg2DADA86oM/bfefvttpaWl6dq1ayotLdXdd999w90+AODmUu2TsdnZ2crOzlb//v21ceNGhYaG1sVcAAAPqTb0wcHBcjqdKi4uVps2bVRSUlIXcwEAPKTa0Ddr1kwZGRkKCAhQcnKyLl++XBdzAQA8xGFZlnWjP+B2u/XNN98oKChImZmZioiIUPv27W0baN3QcNuODfwYxeM3e3sEoEpRXVpWeV+1T8auW7eu4nbjxo21Z88eW0MPAPCsakN/5MgRSZJlWdq/f7+Cg4M1cOBAu+cCAHhItaEfN25cxW3LsjRixAhbBwIAeFa1oS8rK6u4XVhYqFOnTtk6EADAs6oNfd++feVwOGRZlvz9/fXCCy/YOhBPeAGAZ1Ub+jlz5qhTp04VX+/cudPWgQAAnlVl6P/1r3/p8OHDWrJkiX77299Kuv5SyxUrVmj9+vV1NiAA4MepMvSBgYE6c+aMysrKVFhYKElyOBwaP358nQ0HAPjxqn3DVH5+vs6dO6fw8HB98MEHeuSRR+Tn52fbQO9+zpO9AFBbN3rDVLWXQJg+fbq++OILSVJubq7i4+M9NhgAwH7Vhj4/P19RUVGSpOHDh3M9egC4xdToM2Nzc3MlScePH5fb7bZ1IACAZ1X78srJkycrLi5OZ8+elb+/v55++um6mAsA4CHV7ug7d+6sadOm6ec//7lKSkp09uzZupgLAOAhVe7oy8rKtGHDBq1YsUJOp1OXL1/Wli1b5O/vX5fzAQB+pCp39I899pgOHDigpKQkpaWl6a677iLyAHALqnJHP2zYMK1fv15ff/21Bg0apGpebg8AuElV+4apnTt3Kj09XdnZ2Ro0aJAGDBigjh072jYQb5gCgNq70Rumqg39ty5evKi1a9dq1apVWrNmjadm+x5CDwC155HQ1xVCDwC196MugQAAuLURegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwHKE32OUL5/XmyGdV+PUJb48CVCh3uZTx9nQtnPqSUl/9T/5+1gFCb6hyl0vrF86Wr9Pp7VGASg59sUNud7lemPZ/9cj/iVHWe6neHsl4hN5Q/++deerau78aN2nq7VGASu5o3lLu8nK53W5dLbkin3r1vD2S8XztOvDJkye1detWXb16teJ7w4cPt2s5fMfnH25Sg8Bgte/cTdvWvOvtcYBKnPUDVFSYp7+MjdWVSxc1dMJ0b49kPNt29CNHjtSFCxfkdDor/kHd+PzDTTr65ada/NpY5R0/rMyURF0qOuftsQBJ0n9tzFD7zt00es4y/X7mfK2ZO1PXysq8PZbRbNvRN2/eXC+99JJdh8cNPJcwp+L24tfG6qkX4tQ4OMR7AwHfEdCwsXx8r6cnoFFjlbtcstzlXp7KbLaFvmfPnkpKSlL79u0rvjdw4EC7lgNwi+jx5CCtnTdLqa/+p8pdLvV69nk5/QO8PZbRHJZlWXYcOCYmRmFhYQoMDLy+kMOhsWPHVvu4dz8/Zcc4AGC0qC4tq7zPth290+nUa6+9ZtfhAQA1ZFvoW7Roob/97W+699575XA4JEmRkZF2LQcAqIJtoXe5XDp27JiOHTtW8T1CDwB1z7bQz5gxo9LXBQUFdi0FALgB20L/9ttvKy0tTdeuXVNpaanuvvtubdiwwa7lAABVsO0NU9nZ2crOzlb//v21ceNGhYaG2rUUAOAGbAt9cHCwnE6niouL1aZNG5WUlNi1FADgBmwLfbNmzZSRkaGAgAAlJSXp8uXLdi0FALgB294w5Xa7lZeXp8DAQGVmZioiIqLSu2SrwhumAKD2vPKGqfPnzys1NVXHjh1Thw4ddOedd9q1FADgBmw7dRMXF6ewsDC9/PLLatmypSZMmGDXUgCAG7BtRy9JQ4cOlSTdc8892rRpk51LAQCqYNuOPiwsTGvXrlV+fr6ysrIUHBys3Nxc5ebm2rUkAOAH2Hr1Skm6ePGi6tWrp4YNG15f0OHQsmXLqnwcT8YCQO3d6MlYj+/o9+7dq4EDB2rRokWKiYlRYWGhiouLFRsbq+XLl98w8gAAz/N46GfPnq3ExEQ5nU7NmTNHCxYs0KpVq7RgwQJPLwUAqAGPPxlrWZbuuece5efnq6SkRD/96U8lqeJSxQCAuuXxHb3b7ZYkbdu2TREREZKksrIyXblyxdNLAQBqwOM7+oiICD377LPKy8vT3LlzdeLECSUkJKhfv36eXgoAUAO2vOrmyJEjCgkJUZMmTXTixAkdOHBAffr0qdFjedUNANRenV8CoV27dhW3W7durdatW9uxDACgBmx7wxQA4OZA6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAxH6AHAcIQeAAznsCzL8vYQAAD7sKMHAMMRegAwHKEHAMMRegAwHKEHAMMRegAwnK+3B8B1O3bs0KhRo/T++++refPmkqSkpCSFhYXpmWee+cHHFBUVadu2berfv3+l7x8/flzTp09XeXm5XC6X7rvvPo0bN04+Pvx3HfaaP3++PvnkE/n4+MjhcGjMmDG67777vD3WbY9/828ifn5+mjRpkmr61oYDBw4oKyvre99/8803FR0drUWLFmnJkiU6duyYtmzZ4ulxgUoOHz6srKwsLV68WKmpqXr55Zc1efJkb48FsaO/qfTo0UNut1srVqxQdHR0pftSU1O1YcMG+fr6qmvXrho/frzmzZunr776Su+9956GDBlS8WdbtGihzMxMNWzYUJ06ddKcOXPk6+urHTt2aN68efLx8VFhYaGGDBmiX//619q5c6f+8pe/SJJKS0s1c+ZM+fn5acyYMWrevLlOnTqlJ598UocOHdK+ffv06KOPauzYsXX6u8HNLyQkRKdPn1ZGRoYefvhhhYeHKyMjQzExMWrbtq1yc3NlWZZmz56tkJAQ/fGPf1ReXp7Onz+vhx9+WHFxcYqPj5evr69Onz6tsrIy9evXT1u3btU333yjlJQUtW7d2ts/5q3Jwk0hJyfHiouLs86dO2f16tXLys3NtWbNmmWtWrXK+uqrr6xBgwZZZWVlltvttkaNGmVlZWVVPOZ/u3r1qrV48WJr6NChVteuXa1x48ZZFy5csHJycqwnnnjCunr1qlVSUmL17t3bOnPmjPXOO+9YeXl5lmVZ1ty5c62UlBTr5MmTVvfu3a2LFy9aBQUF1v3332+dP3/eKi0ttSIiIur614NbxJ49e6z4+HjrkUcesX75y19amzZtsqKjo63MzEzLsizrnXfesaZNm2adPHnS+vvf/25ZlmWVlpZaDz74oGVZljVx4kQrJSXFsizLmjp1qjVz5kzLsizrrbfeshYvXlznP48p2NHfZJo0aaLJkycrPj5eDzzwgCTp6NGj6ty5s/z8/CRJXbt21aFDh9S5c+cfPEZOTo5iY2MVGxur4uJizZw5UykpKerZs6e6dOkip9MpSerQoYNOnDih0NBQTZ8+XQ0aNFB+fn7Fuq1atVLjxo3ldDrVtGlTBQcHS5IcDofNvwXcio4fP65GjRppxowZkqQvv/xSL774opo2baoePXpIkh544AFlZWUpODhYX375pXJyctSoUSOVlZVVHOfee++VJAUGBiosLKzi9nf/DGqHc/Q3occee0xt27ZVZmamJCksLEy7d++Wy+WSZVnatWuX2rZtKx8fH7nd7u89ftasWfr4448lSQ0bNlTbtm0r4r5//36Vl5erpKREhw8fVps2bTRlyhS98cYbSkxM1F133VXxHAFBR20cOHBACQkJunr1qiSpbdu2aty4serVq6c9e/ZIkj777DO1b99eq1evVuPGjZWcnKznnntOpaWl/L2zETv6m9Qrr7yinJwcSdJPfvITPfHEE4qKipLb7dbPfvYz9e7dWwUFBTp48KCWLFmi2NjYisfOmTNHf/rTn5ScnCyn06mWLVsqISFBe/fulcvl0vDhw1VUVKTf//73CgkJ0YABAzR48GAFBgaqadOmKigo8NJPjVvZ448/riNHjuhXv/qVGjRoIMuyNGHCBC1dulSZmZlasmSJAgIC9Oc//1lnzpzR2LFj9emnnyogIEBt2rTh752NuHrlbWTHjh1auXKlZs+e7e1RcBuJiYlRQkKC2rVr5+1RblucugEAw7GjBwDDsaMHAMMRegAwHKEHAMMRehhrx44dioiIUExMjGJiYjR48GAtX7681sdJSkrS6tWrtX///opLRfyQzZs3Kz8/v0bHzM7OVnx8fK1nAf4dvI4eRuvRo0fFy0nLysrUt29fDRgwQIGBgbU+Vnh4uMLDw6u8f9myZUpISFBoaOi/PS9gB0KP28bly5fl4+Oj2NhYtWzZUhcvXtT8+fOVkJCg48ePy+12Ky4uTt27d9c///lPzZ07VyEhIbp27ZrCwsIqvQ8hPT1d7777rtxut3r16qX7779f+/fv18SJE5WWlqb33ntP69evl8PhUL9+/TRs2DAdOXJEkydPVkBAgAICAhQUFOTtXwluE4QeRsvJyVFMTIwcDof8/Pw0depULVy4UP3791efPn2UlpamJk2a6I033tD58+cVHR2tDRs2aNasWUpPT1dwcLBefPHFSsc8e/asFixYoHXr1snpdCoxMVHdunVTeHi4EhISdOLECW3cuFFpaWlyOByKjY1VZGSk3nrrLY0ePVoPPfSQ5s+fr6NHj3rpt4LbDaGH0b576uZbCxcuVNu2bSVJBw8e1Keffqrdu3dLklwul86cOaNGjRqpSZMmkqQuXbpUevzJkyfVoUMH+fv7S9L3rrl+8OBBnT59uuKyFBcuXNCJEyd06NAhderUSdL1i3sRetQVnozFbenbC2eFhYXpySef1PLly7VgwQL17dtXgYGBunTpks6dOyfp+lUYv6t169Y6evRoxdUUR48erfz8fDkcDlmWpbCwMLVv317Lli3T8uXL9cwzz6hjx44KCwvT559/LkkVF/kC6gI7etzWnn32WU2ZMkXR0dG6fPmyhg4dKqfTqRkzZuj5559XUFCQfH0r/2sSEhKi4cOHKzo6Wg6HQz179lRoaKi6dOmiCRMmKDU1VREREYqKilJZWZk6deqk0NBQvfrqqxozZowWLVqkkJAQ1a9f30s/NW43XAIBAAzHqRsAMByhBwDDEXoAMByhBwDDEXoAMByhBwDDEXoAMByhBwDD/X9aoQT+C3CaSwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "confusion_matrix = pd.crosstab(y_test, pred, rownames=['Actual'], colnames=['Predicted'])\n",
    "plt.figure(figsize = (6, 6))\n",
    "sns.heatmap(confusion_matrix, annot = True, cmap = 'Paired', cbar = False, fmt=\"d\", xticklabels=['Not Spam', 'Spam'], yticklabels=['Not Spam', 'Spam']);"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}