NaiveBayes_InDepth.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d279f05b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:95% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline \n",
    "%config Completer.use_jedi = False\n",
    "\n",
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:95% !important; }</style>\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "473a69ef",
   "metadata": {},
   "source": [
    "# Tasks \n",
    "\n",
    "- [x] Explain Simple and Optimized Implementations of the Algorithm Step by Step"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39feee54",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ec2c02d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from drawdata import draw_scatter\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import plotly as pl\n",
    "import plotly.express as px\n",
    "import plotly.graph_objects as go\n",
    "import math\n",
    "from pathlib import Path\n",
    "import matplotlib.pyplot as plt\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "from tqdm import tqdm\n",
    "from tqdm.notebook import tqdm\n",
    "import gc\n",
    "import re\n",
    "from collections import defaultdict\n",
    "from typing import List, Union\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler, Normalizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "425e6b34",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "pl.io.renderers.default = 'iframe_connected'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1b71a6b",
   "metadata": {
    "tags": [],
    "toc-hr-collapsed": true
   },
   "source": [
    "# Bayes' Theorom Python Implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a594f0b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def bayes_theorom(p_a, p_b_given_a, p_b_given_not_a):\n",
    "    not_a = 1 - p_a\n",
    "    p_b = p_b_given_a * p_a + p_b_given_not_a * not_a\n",
    "    p_a_given_b = (p_b_given_a * p_a)/p_b\n",
    "    return p_a_given_b"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53b4c706",
   "metadata": {
    "tags": [],
    "toc-hr-collapsed": true
   },
   "source": [
    "# Naive Bayes Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dce0bb52",
   "metadata": {},
   "source": [
    "## Dummy Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "be2f122b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_blobs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1d8a3471",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((100, 2), (100,))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X, y = make_blobs(n_samples=100, centers=2, n_features=2, random_state=1)\n",
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4aa2e7d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.79415228,  2.10495117],\n",
       "       [-9.15155186, -4.81286449],\n",
       "       [-3.10367371,  3.90202401],\n",
       "       [-1.42946517,  5.16850105],\n",
       "       [-7.4693868 , -4.20198333]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c1b5d165",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 0, 0, 1])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "18536f11",
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import norm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4170986f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# fit a probability distribution to a univariate data sample\n",
    "def fit_distribution(data):\n",
    "    mu = np.mean(data)\n",
    "    sigma = np.std(data)\n",
    "    print(mu, sigma)\n",
    "    dist = norm(mu, sigma)\n",
    "    return dist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "83396653",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-5.6222329955827375 4.145474011953628\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<scipy.stats._distn_infrastructure.rv_frozen at 0x130058b8820>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fit_distribution(X[:,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "41fd4b5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# data by class\n",
    "Xy0 = X[y==0]\n",
    "Xy1 = X[y==1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3207fb8e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((50, 2), (50, 2))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xy0.shape, Xy1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ecba1d72",
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate the priors \n",
    "prior0 = len(Xy0)/len(X)\n",
    "prior1 = len(Xy1)/len(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "acdde030",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.5, 0.5)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prior0, prior1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "194cb310",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-1.5632888906409914 0.787444265443213\n",
      "4.426680361487157 0.958296071258367\n"
     ]
    }
   ],
   "source": [
    "X0y0= fit_distribution(Xy0[:,0])\n",
    "X1y0 = fit_distribution(Xy0[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b75a26f2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-9.681177100524485 0.8943078901048118\n",
      "-3.9713794295185845 0.9308177595208521\n"
     ]
    }
   ],
   "source": [
    "X0y1= fit_distribution(Xy1[:,0])\n",
    "X1y1 = fit_distribution(Xy1[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "67d0813b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def probability(X, prior, dist1, dist2):\n",
    "    return prior * dist1.pdf(X[0]) * dist2.pdf(X[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c861985f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# classify one example\n",
    "Xsample, ysample = X[0], y[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a32512dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "py0 = probability(Xsample, prior0, X0y0, X1y0)\n",
    "py1 = probability(Xsample, prior1, X0y1, X1y1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "49b28b7d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P(y=0 | [-0.79415228  2.10495117]) = 0.348\n",
      "P(y=1 | [-0.79415228  2.10495117]) = 0.000\n"
     ]
    }
   ],
   "source": [
    "print('P(y=0 | %s) = %.3f' % (Xsample, py0*100))\n",
    "print('P(y=1 | %s) = %.3f' % (Xsample, py1*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ad5d7598",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ysample"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b991802b",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Implement from Scratch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be4a0261",
   "metadata": {
    "tags": [],
    "toc-hr-collapsed": true
   },
   "source": [
    "### Load IRIS Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "9c883142",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "be7ba505",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = datasets.load_iris()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "e472549a",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = data['data']\n",
    "y = data['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "3a9b84b3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150, 4)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "80fab28f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5.1, 3.5, 1.4, 0.2]])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[:1, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "a3b69a6f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['sepal length (cm)',\n",
       " 'sepal width (cm)',\n",
       " 'petal length (cm)',\n",
       " 'petal width (cm)']"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['feature_names']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "4a986d6e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['data', 'target', 'frame', 'target_names', 'DESCR', 'feature_names', 'filename'])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "b2b4c2a5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['setosa', 'versicolor', 'virginica'], dtype='<U10')"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['target_names']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "e802b181",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150, 4)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "2f480ef5",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = np.concatenate((X, y.reshape(-1,1)), axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "025f220d",
   "metadata": {},
   "source": [
    "## Gaussian NB "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "345a1a58",
   "metadata": {
    "tags": [],
    "toc-hr-collapsed": true
   },
   "source": [
    "### 1. Separate by class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "6e6b007f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0.0, 1.0, 2.0}"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(data[:,-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "3935e361",
   "metadata": {},
   "outputs": [],
   "source": [
    "def separate_by_class(data):\n",
    "    separated = dict()\n",
    "    classes = set(data[:,-1])\n",
    "    for c in classes:\n",
    "        separated[c] = data[data[:,-1]==c]\n",
    "    return separated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "e17391d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([0.0, 1.0, 2.0])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "separated = separate_by_class(data)\n",
    "separated.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8278d25b",
   "metadata": {
    "tags": [],
    "toc-hr-collapsed": true
   },
   "source": [
    "### 2. Summarize Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "9dbc9eee",
   "metadata": {},
   "outputs": [],
   "source": [
    "from math import sqrt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "cfb97c62",
   "metadata": {},
   "outputs": [],
   "source": [
    "def mean(numbers):\n",
    "    return sum(numbers)/float(len(numbers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "94e408f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def std(numbers):\n",
    "    avg = mean(numbers)\n",
    "    variance =sum([(x - avg)**2 for x in numbers])/float(len(numbers)-1)\n",
    "    return np.sqrt(variance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "721b99e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(5.843333333333335, 0.8280661279778629, 150),\n",
       " (3.057333333333334, 0.435866284936698, 150),\n",
       " (3.7580000000000027, 1.7652982332594667, 150),\n",
       " (1.199333333333334, 0.7622376689603465, 150)]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(mean(data[:, col]), std(data[:, col]), len(data[:, col])) for col in range(data.shape[1]-1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "ffb24367",
   "metadata": {},
   "outputs": [],
   "source": [
    "def summarize_dataset(data):\n",
    "    summaries = [(mean(data[:, col]), std(data[:, col]), len(data[:, col])) for col in range(data.shape[1]-1)]\n",
    "    return summaries"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1689264c",
   "metadata": {
    "tags": [],
    "toc-hr-collapsed": true
   },
   "source": [
    "### 3. Summarize Data by Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "61114a9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def summarize_by_class(data):\n",
    "    separated = separate_by_class(data)\n",
    "    summaries = dict()\n",
    "    for label, rows in separated.items():\n",
    "        summaries[label] = summarize_dataset(rows)\n",
    "    return summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "68f7e3a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "summary = summarize_by_class(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "1267b2ec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0.0: [(5.005999999999999, 0.3524896872134512, 50),\n",
       "  (3.428000000000001, 0.3790643690962886, 50),\n",
       "  (1.4620000000000002, 0.1736639964801841, 50),\n",
       "  (0.2459999999999999, 0.10538558938004569, 50)],\n",
       " 1.0: [(5.936, 0.5161711470638635, 50),\n",
       "  (2.7700000000000005, 0.3137983233784114, 50),\n",
       "  (4.26, 0.46991097723995806, 50),\n",
       "  (1.3259999999999998, 0.197752680004544, 50)],\n",
       " 2.0: [(6.587999999999998, 0.635879593274432, 50),\n",
       "  (2.9739999999999998, 0.3224966381726376, 50),\n",
       "  (5.552, 0.5518946956639835, 50),\n",
       "  (2.026, 0.27465005563666733, 50)]}"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "822dcaee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary[0][0][2]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "262bd352",
   "metadata": {
    "tags": [],
    "toc-hr-collapsed": true
   },
   "source": [
    "### 4. Gaussian Probability Distribution Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "1ef1163c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_gaussian_probability(x, mean, std):\n",
    "    exponent = np.exp(-((x-mean)**2/(2*std**2)))\n",
    "    return (1/(np.sqrt(2*np.pi)*std)) * exponent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "297bf7b4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3989422804014327"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calculate_gaussian_probability(1, 1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "11553b8e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.24197072451914337, 0.24197072451914337)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calculate_gaussian_probability(2, 1, 1), calculate_gaussian_probability(0, 1, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c9d0504",
   "metadata": {
    "tags": [],
    "toc-hr-collapsed": true
   },
   "source": [
    "### Class Probabilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "bc262480",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_class_probabilities(summaries, row):\n",
    "    total_rows = sum([summaries[label][0][2] for label in summaries])\n",
    "    probabilities = dict()\n",
    "    for label, class_summaries in summaries.items():\n",
    "        probabilities[label] = summaries[label][0][2]/float(total_rows)\n",
    "        for i in range(len(class_summaries)):\n",
    "            mean, std, count =  class_summaries[i]\n",
    "            probabilities[label] *= calculate_gaussian_probability(row[i], mean, std)\n",
    "    return probabilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "b47a1c77",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = data[0][:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "3acb6c61",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5.1, 3.5, 1.4, 0.2])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "1ed76bad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0.0: 2.7915339171768885,\n",
       " 1.0: 8.322426199968131e-18,\n",
       " 2.0: 6.008422572010989e-25}"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calculate_class_probabilities(summary, sample)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a43a206",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Gaussian NB Optimized"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a64bd771",
   "metadata": {
    "tags": [],
    "toc-hr-collapsed": true
   },
   "source": [
    "### Calculation Tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "1819db1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.DataFrame(data, columns=['sepal length', 'sepal width', 'petal length', 'petal width', 'target'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "e50526d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "m, v = data.groupby('target').apply(np.mean).values[:,:-1], data.groupby('target').apply(np.var).values[:,:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "8909dddd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5.006, 3.428, 1.462, 0.246],\n",
       "       [5.936, 2.77 , 4.26 , 1.326],\n",
       "       [6.588, 2.974, 5.552, 2.026]])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "49a7e485",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.121764, 0.140816, 0.029556, 0.010884],\n",
       "       [0.261104, 0.0965  , 0.2164  , 0.038324],\n",
       "       [0.396256, 0.101924, 0.298496, 0.073924]])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "e9683d21",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1.0986122986681097"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.log(0.33333333)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "e7417587",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.log(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "9e20fae7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.4965075614664802"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.log(33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "f9c8ca60",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33.0"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.exp(3.4965075614664802)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "67256ffa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# np.sum(np.log((np.exp((-1/2)*((samples.iloc[0,:-1].values-m)**2)/(2*v))))[0])+np.log(0.33333333)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a75c7356",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.sum(np.log((np.exp((-1/2)*((samples.iloc[0,:-1].values-m)**2)/(2*v))))[1])+np.log(0.33333333)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8eb77b34",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.sum(np.log((np.exp((-1/2)*((samples.iloc[0,:-1].values-m)**2)/(2*v))))[2])+np.log(0.33333333)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7b9a4ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "stats[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b75356b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "stats[0][1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f17187f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "(data.groupby('target')[data.columns[0]].count()/data.shape[0]).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e019bfe1",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(data.iloc[0,:-1].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2bf532f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "data['target'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0852eb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "data.target.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e872245",
   "metadata": {},
   "source": [
    "### GNB with Numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "dd08082c",
   "metadata": {},
   "outputs": [],
   "source": [
    "class GaussianNBCustom():\n",
    "    def __init__(self,):\n",
    "        pass\n",
    "    \n",
    "    def fit(self, data, target):\n",
    "        \"\"\" calculate mean and variance of each feature per class\n",
    "            data: pd.DataFrame\n",
    "            target: target column name\n",
    "        \"\"\"\n",
    "        self.classes = data['target'].unique()\n",
    "        \n",
    "        self.mean, self.var = data.groupby('target').apply(np.mean).values[:,:-1], data.groupby('target').apply(np.var).values[:,:-1]\n",
    "        \n",
    "        self.prior = (data.groupby('target')[data.columns[0]].count()/data.shape[0]).values\n",
    "    \n",
    "    def gaussian_pdf(self, class_idx, x):\n",
    "        mean, var = self.mean[class_idx], self.var[class_idx]\n",
    "        numerator = np.exp((-1/2)*((x-mean)**2)/(2*var))\n",
    "        denominator = np.sqrt(2*np.pi*var)\n",
    "        prob = numerator/denominator\n",
    "        return prob\n",
    "    \n",
    "    def calc_posterior(self, x):\n",
    "        posteriors = []\n",
    "        for i in range(len(self.classes)):\n",
    "            prior = np.log(self.prior[i])\n",
    "            conditional = np.sum(np.log(self.gaussian_pdf(i, x)))\n",
    "            posterior = prior + conditional\n",
    "            posteriors.append(posterior)\n",
    "        \n",
    "        return self.classes[np.argmax(posteriors)]\n",
    "        \n",
    "    def predict(self, X):\n",
    "        preds = [self.calc_posterior(x) for x in X.values]\n",
    "        return preds        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "0ebe5522",
   "metadata": {},
   "outputs": [],
   "source": [
    "gnb = GaussianNBCustom()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "6d994a2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "gnb.fit(data, 'target')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "6d91d8e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "samples = data.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "01e6e369",
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = gnb.predict(samples.iloc[:,:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "c2be9e37",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[True, True, True, True, True, True, True, True, True, True]"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(x==y) for x, y in zip(preds, samples.target.tolist())]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "169dc7bb",
   "metadata": {},
   "source": [
    "## Multinomial Naive Bayes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1d326da",
   "metadata": {},
   "source": [
    "### Get Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "25d42d60",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "WindowsPath('D:/amit/ml_indepth/naivebayes_mle_map')"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Path.cwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "97b6886f",
   "metadata": {},
   "outputs": [],
   "source": [
    "sms_df = pd.read_csv('../datasets/SMSSpamCollection', header=None, sep='\\t', names=['label', 'sms'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "d7ea36a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>sms</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5567</th>\n",
       "      <td>spam</td>\n",
       "      <td>This is the 2nd time we have tried 2 contact u...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5568</th>\n",
       "      <td>ham</td>\n",
       "      <td>Will ü b going to esplanade fr home?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5569</th>\n",
       "      <td>ham</td>\n",
       "      <td>Pity, * was in mood for that. So...any other s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5570</th>\n",
       "      <td>ham</td>\n",
       "      <td>The guy did some bitching but I acted like i'd...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5571</th>\n",
       "      <td>ham</td>\n",
       "      <td>Rofl. Its true to its name</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5572 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                                sms\n",
       "0      ham  Go until jurong point, crazy.. Available only ...\n",
       "1      ham                      Ok lar... Joking wif u oni...\n",
       "2     spam  Free entry in 2 a wkly comp to win FA Cup fina...\n",
       "3      ham  U dun say so early hor... U c already then say...\n",
       "4      ham  Nah I don't think he goes to usf, he lives aro...\n",
       "...    ...                                                ...\n",
       "5567  spam  This is the 2nd time we have tried 2 contact u...\n",
       "5568   ham               Will ü b going to esplanade fr home?\n",
       "5569   ham  Pity, * was in mood for that. So...any other s...\n",
       "5570   ham  The guy did some bitching but I acted like i'd...\n",
       "5571   ham                         Rofl. Its true to its name\n",
       "\n",
       "[5572 rows x 2 columns]"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sms_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "3c701476",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ham     0.865937\n",
       "spam    0.134063\n",
       "Name: label, dtype: float64"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sms_df.label.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6eb7058e",
   "metadata": {},
   "source": [
    "### Split Train Test"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47a2160b",
   "metadata": {},
   "source": [
    "#### Manually"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "60f4e6e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "sms_randomized = sms_df.sample(frac=1, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "df3ea9a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4458"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_test_idx = round(len(sms_randomized)*0.8)\n",
    "training_test_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "16c78db4",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set = sms_randomized[:training_test_idx].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "095d7678",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_set = sms_randomized[training_test_idx:].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "fd73068d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((4458, 2), (1114, 2))"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.shape, test_set.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "2ba4490d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ham     0.866981\n",
       "spam    0.133019\n",
       "Name: label, dtype: float64"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.label.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "69222393",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ham     0.861759\n",
       "spam    0.138241\n",
       "Name: label, dtype: float64"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_set.label.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "376492a3",
   "metadata": {},
   "source": [
    "#### Stratified Shuffle Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "9ab2b42e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedShuffleSplit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "38621066",
   "metadata": {},
   "outputs": [],
   "source": [
    "sss = StratifiedShuffleSplit(n_splits=5, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "03373304",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sss.get_n_splits(sms_df.sms.values, sms_df.label.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "6735efa6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRAIN: [ 184 2171 5422 ... 2309 1904  762] TEST: [2825 3695 3904 ... 2015 3380  785]\n",
      "TRAIN: [3535 3026 1592 ... 3319  545 2131] TEST: [1573  966 4323 ... 3427 3784 2526]\n",
      "TRAIN: [2528  635  699 ...  485 2784 3898] TEST: [1113 5041 5065 ... 3516 2366 1821]\n",
      "TRAIN: [1011 3634 1222 ... 2441 4439 2403] TEST: [5059 1488 4018 ...  496 4096 2110]\n",
      "TRAIN: [ 752 4008 5511 ... 4794 1155 1479] TEST: [1115 3969 3191 ...   31 2069 2314]\n"
     ]
    }
   ],
   "source": [
    "for train_index, test_index in sss.split(sms_df.sms.values, sms_df.label.values):\n",
    "    print(\"TRAIN:\", train_index, \"TEST:\", test_index)\n",
    "    X_train, X_test = sms_df.sms.values[train_index], sms_df.sms.values[test_index]\n",
    "    y_train, y_test = sms_df.label.values[train_index], sms_df.label.values[test_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "4e14d80a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((4457,), (4457,))"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape, y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "e982fa28",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1115,), (1115,))"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "6c61a2e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "de9f0bd6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'spam': 0.13417, 'ham': 0.86583}"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "{k:round(v/len(y_train),5) for k, v in Counter(y_train).items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "df51ef45",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set = pd.DataFrame(zip(X_train, y_train), columns=['sms', 'label'])\n",
    "test_set = pd.DataFrame(zip(X_test, y_test), columns=['sms', 'label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "dbb68fa8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((4457, 2), (1115, 2))"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.shape, test_set.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "ff36af2f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ham     0.865829\n",
       "spam    0.134171\n",
       "Name: label, dtype: float64"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.label.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "9ddf3282",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ham     0.866368\n",
       "spam    0.133632\n",
       "Name: label, dtype: float64"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_set.label.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b6797aa",
   "metadata": {},
   "source": [
    "### Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6be6025b",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set.rename(columns={'sms':'TEXT'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "id": "9635993e",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set.TEXT = train_set.TEXT.str.lower().str.replace('\\W', ' ', regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "id": "6fd13cc0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'you have an important customer service announcement from premier '"
      ]
     },
     "execution_count": 303,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.TEXT.values[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "id": "961fe874",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 10.3 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "vocab = set(' '.join(train_set.TEXT.values).replace('\\W', ' ').split())\n",
    "vocab = list(vocab - set(stopwords.words('english')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "id": "88f9f557",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7670"
      ]
     },
     "execution_count": 306,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "d8e4dc28",
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx, sms in enumerate(train_set.sms):\n",
    "    for word in sms.split():\n",
    "        word_counts_per_sms[word][idx] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "id": "e7dac6f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "word_counts = pd.DataFrame(word_counts_per_sms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "id": "b8fedf13",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cs</th>\n",
       "      <th>cricket</th>\n",
       "      <th>serena</th>\n",
       "      <th>hit</th>\n",
       "      <th>sing</th>\n",
       "      <th>linerental</th>\n",
       "      <th>enters</th>\n",
       "      <th>meet</th>\n",
       "      <th>ate</th>\n",
       "      <th>changes</th>\n",
       "      <th>...</th>\n",
       "      <th>norm150p</th>\n",
       "      <th>prometazine</th>\n",
       "      <th>09066362220</th>\n",
       "      <th>med</th>\n",
       "      <th>sportsx</th>\n",
       "      <th>scool</th>\n",
       "      <th>poop</th>\n",
       "      <th>toss</th>\n",
       "      <th>bribe</th>\n",
       "      <th>vijaykanth</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4452</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4453</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4454</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4455</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4456</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4457 rows × 7670 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      cs  cricket  serena  hit  sing  linerental  enters  meet  ate  changes  \\\n",
       "0      0        0       0    0     0           0       0     0    0        0   \n",
       "1      0        0       0    0     0           0       0     0    0        0   \n",
       "2      0        0       0    0     0           0       0     0    0        0   \n",
       "3      0        0       0    0     0           0       0     0    0        0   \n",
       "4      0        0       0    0     0           0       0     0    0        0   \n",
       "...   ..      ...     ...  ...   ...         ...     ...   ...  ...      ...   \n",
       "4452   0        0       0    0     0           0       0     0    0        0   \n",
       "4453   0        0       0    0     0           0       0     0    0        0   \n",
       "4454   0        0       0    0     0           0       0     0    0        0   \n",
       "4455   0        0       0    0     0           0       0     0    0        0   \n",
       "4456   0        0       0    0     0           0       0     0    0        0   \n",
       "\n",
       "      ...  norm150p  prometazine  09066362220  med  sportsx  scool  poop  \\\n",
       "0     ...         0            0            0    0        0      0     0   \n",
       "1     ...         0            0            0    0        0      0     0   \n",
       "2     ...         0            0            0    0        0      0     0   \n",
       "3     ...         0            0            0    0        0      0     0   \n",
       "4     ...         0            0            0    0        0      0     0   \n",
       "...   ...       ...          ...          ...  ...      ...    ...   ...   \n",
       "4452  ...         0            0            0    0        0      0     0   \n",
       "4453  ...         0            0            0    0        0      0     0   \n",
       "4454  ...         0            0            0    0        0      0     0   \n",
       "4455  ...         0            0            0    0        0      0     0   \n",
       "4456  ...         0            0            0    0        0      0     0   \n",
       "\n",
       "      toss  bribe  vijaykanth  \n",
       "0        0      0           0  \n",
       "1        0      0           0  \n",
       "2        0      0           0  \n",
       "3        0      0           0  \n",
       "4        0      0           0  \n",
       "...    ...    ...         ...  \n",
       "4452     0      0           0  \n",
       "4453     0      0           0  \n",
       "4454     0      0           0  \n",
       "4455     0      0           0  \n",
       "4456     0      0           0  \n",
       "\n",
       "[4457 rows x 7670 columns]"
      ]
     },
     "execution_count": 311,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "id": "0195ef72",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TEXT</th>\n",
       "      <th>label</th>\n",
       "      <th>n_words_in_text</th>\n",
       "      <th>cs</th>\n",
       "      <th>cricket</th>\n",
       "      <th>serena</th>\n",
       "      <th>hit</th>\n",
       "      <th>sing</th>\n",
       "      <th>linerental</th>\n",
       "      <th>enters</th>\n",
       "      <th>...</th>\n",
       "      <th>norm150p</th>\n",
       "      <th>prometazine</th>\n",
       "      <th>09066362220</th>\n",
       "      <th>med</th>\n",
       "      <th>sportsx</th>\n",
       "      <th>scool</th>\n",
       "      <th>poop</th>\n",
       "      <th>toss</th>\n",
       "      <th>bribe</th>\n",
       "      <th>vijaykanth</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>you have an important customer service announc...</td>\n",
       "      <td>spam</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>i m reaching home in 5 min</td>\n",
       "      <td>ham</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>it s reassuring  in this crazy world</td>\n",
       "      <td>ham</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>are you staying in town</td>\n",
       "      <td>ham</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>you will go to walmart  i ll stay</td>\n",
       "      <td>ham</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 7673 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                TEXT label  n_words_in_text  \\\n",
       "0  you have an important customer service announc...  spam                9   \n",
       "1                        i m reaching home in 5 min    ham                7   \n",
       "2              it s reassuring  in this crazy world    ham                7   \n",
       "3                          are you staying in town     ham                5   \n",
       "4                 you will go to walmart  i ll stay    ham                8   \n",
       "\n",
       "   cs  cricket  serena  hit  sing  linerental  enters  ...  norm150p  \\\n",
       "0   0        0       0    0     0           0       0  ...         0   \n",
       "1   0        0       0    0     0           0       0  ...         0   \n",
       "2   0        0       0    0     0           0       0  ...         0   \n",
       "3   0        0       0    0     0           0       0  ...         0   \n",
       "4   0        0       0    0     0           0       0  ...         0   \n",
       "\n",
       "   prometazine  09066362220  med  sportsx  scool  poop  toss  bribe  \\\n",
       "0            0            0    0        0      0     0     0      0   \n",
       "1            0            0    0        0      0     0     0      0   \n",
       "2            0            0    0        0      0     0     0      0   \n",
       "3            0            0    0        0      0     0     0      0   \n",
       "4            0            0    0        0      0     0     0      0   \n",
       "\n",
       "   vijaykanth  \n",
       "0           0  \n",
       "1           0  \n",
       "2           0  \n",
       "3           0  \n",
       "4           0  \n",
       "\n",
       "[5 rows x 7673 columns]"
      ]
     },
     "execution_count": 312,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set_words = pd.concat([train_set, word_counts], axis=1)\n",
    "train_set_words.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "id": "adafefbe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>n_words_in_text</th>\n",
       "      <th>cs</th>\n",
       "      <th>cricket</th>\n",
       "      <th>serena</th>\n",
       "      <th>hit</th>\n",
       "      <th>sing</th>\n",
       "      <th>linerental</th>\n",
       "      <th>enters</th>\n",
       "      <th>meet</th>\n",
       "      <th>ate</th>\n",
       "      <th>...</th>\n",
       "      <th>norm150p</th>\n",
       "      <th>prometazine</th>\n",
       "      <th>09066362220</th>\n",
       "      <th>med</th>\n",
       "      <th>sportsx</th>\n",
       "      <th>scool</th>\n",
       "      <th>poop</th>\n",
       "      <th>toss</th>\n",
       "      <th>bribe</th>\n",
       "      <th>vijaykanth</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ham</th>\n",
       "      <td>57196</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spam</th>\n",
       "      <td>15148</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 7671 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       n_words_in_text  cs  cricket  serena  hit  sing  linerental  enters  \\\n",
       "label                                                                        \n",
       "ham              57196   0        0       0    0     0           0       0   \n",
       "spam             15148   0        0       0    0     0           0       0   \n",
       "\n",
       "       meet  ate  ...  norm150p  prometazine  09066362220  med  sportsx  \\\n",
       "label             ...                                                     \n",
       "ham       0    0  ...         0            0            0    0        0   \n",
       "spam      0    0  ...         0            0            0    0        0   \n",
       "\n",
       "       scool  poop  toss  bribe  vijaykanth  \n",
       "label                                        \n",
       "ham        0     0     0      0           0  \n",
       "spam       0     0     0      0           0  \n",
       "\n",
       "[2 rows x 7671 columns]"
      ]
     },
     "execution_count": 314,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set_words.groupby('label').sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "id": "0833281d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "spam    0.134171\n",
       "ham     0.865829\n",
       "Name: label, dtype: float64"
      ]
     },
     "execution_count": 313,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set_words.label.value_counts(normalize=True, sort=True, ascending=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "69feb9e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "p_spam, p_ham = train_set_words.label.value_counts(normalize=True, sort=True, ascending=True).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "51abb069",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.13417096701817366, 0.8658290329818263)"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p_spam, p_ham"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "55a6e6fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set_words['n_words_in_text'] = train_set_words.TEXT.str.split().apply(len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "e55a4f1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_ham, n_spam = train_set_words.groupby('label').n_words_in_text.sum().values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "142aa725",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(57196, 15148, 0.8658290329818263, 0.13417096701817366)"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_ham, n_spam, p_ham, p_spam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "7b9e9cb8",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set_words.drop(columns='n_words_in_text', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "a054ce5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_vocab = len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "e00b369e",
   "metadata": {},
   "outputs": [],
   "source": [
    "alpha = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "id": "072a8356",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       you have an important customer service announc...\n",
       "1                             i m reaching home in 5 min \n",
       "2                   it s reassuring  in this crazy world \n",
       "3                               are you staying in town  \n",
       "4                      you will go to walmart  i ll stay \n",
       "                              ...                        \n",
       "4452    claim a 200 shopping spree  just call 08717895...\n",
       "4453    jus telling u dat i ll b leaving 4 shanghai on...\n",
       "4454    or u ask they all if next sat can a not  if al...\n",
       "4455    did u find a sitter for kaitlyn  i was sick an...\n",
       "4456    i think its far more than that but find out  c...\n",
       "Name: TEXT, Length: 4457, dtype: object"
      ]
     },
     "execution_count": 266,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.TEXT"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c80a27b",
   "metadata": {},
   "source": [
    "### Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "id": "9cffb84d",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultinomialNaiveBayesCustom():\n",
    "    def __init__(self, alpha=1):\n",
    "        \"\"\"\n",
    "        alpha: smoothing variable, if \"1\": laplacian smoothing\n",
    "        \"\"\"\n",
    "        self.alpha = alpha\n",
    "\n",
    "    def fit(self, data, X, y):\n",
    "        \"\"\"\n",
    "        data: pandas dataframe containing data\n",
    "        X: feature column name\n",
    "        y: target column name\n",
    "        \"\"\"\n",
    "        self.X, self.y = X, y\n",
    "        self.labels = data[y].unique()\n",
    "\n",
    "        priors = data.label.value_counts(\n",
    "            normalize=True, sort=True, ascending=True)\n",
    "        self.stats = {'p_' + c: priors[c] for c in self.labels}\n",
    "\n",
    "        data['n_words_in_text'] = data[X].str.split().apply(len)\n",
    "\n",
    "        feature_value_counts = data.groupby(y).n_words_in_text.sum()\n",
    "        self.stats.update(\n",
    "            {'n_'+c: feature_value_counts[c] for c in self.labels})\n",
    "\n",
    "        data[X] = data[X].str.lower().str.replace('\\W', ' ', regex=True)\n",
    "\n",
    "        self.vocab = set(' '.join(data[X].values).split())\n",
    "        \n",
    "        self.vocab = list(self.vocab - set(stopwords.words('english')))\n",
    "\n",
    "        self.stats.update({'n_vocab': len(self.vocab)})\n",
    "\n",
    "        word_counts_per_sms = {unique_word: [0] * len(data) for unique_word in self.vocab}\n",
    "\n",
    "        for idx, text in enumerate(data[X]):\n",
    "            for word in text.split():\n",
    "                if word in self.vocab:\n",
    "                    word_counts_per_sms[word][idx] += 1\n",
    "\n",
    "        word_counts = pd.DataFrame(word_counts_per_sms)\n",
    "\n",
    "        self.data = pd.concat([data, word_counts], axis=1)\n",
    "\n",
    "        self.data.drop(columns='n_words_in_text', inplace=True)\n",
    "        del word_counts, priors, word_counts_per_sms, feature_value_counts, data\n",
    "        gc.collect()\n",
    "\n",
    "        self.parameters = self.calc_prob_feature_given_label()\n",
    "\n",
    "    def calc_prob_feature_given_label(self):\n",
    "        label_wise_feature_count = self.data.loc[:, self.data.columns != self.X].groupby(\n",
    "            self.y).sum()\n",
    "        parameters = {'param_'+c: ((label_wise_feature_count.loc[c]+self.alpha)/(\n",
    "            self.stats['n_'+c]+(self.stats['n_vocab']*self.alpha))).to_dict() for c in self.labels}\n",
    "\n",
    "        del label_wise_feature_count\n",
    "        gc.collect()\n",
    "\n",
    "        return parameters\n",
    "\n",
    "    def predict(self, X: Union[str, list]):\n",
    "        \"\"\" predicts the lables of input string or list of strings\n",
    "            X: text messages\n",
    "        \"\"\"\n",
    "        if not isinstance(X, list):\n",
    "            X = list(X)\n",
    "\n",
    "        X = [list(set(re.sub('\\W', ' ', x.lower()).split())) for x in X]\n",
    "\n",
    "        self.test_features_ = X\n",
    "\n",
    "        self.log_probabilities = defaultdict(list)\n",
    "        \n",
    "        {self.log_probabilities[idx].append((np.log(self.stats['p_'+c]) +\n",
    "                                                 sum(np.log([self.parameters['param_'+c][word]\n",
    "                                                             for word in words if word in self.vocab])\n",
    "                                                     ))) for idx, words in enumerate(X) for c in self.labels}\n",
    "\n",
    "        self.predictions = [self.labels[np.argmax(\n",
    "            v)] for k, v in self.log_probabilities.items()]\n",
    "\n",
    "        return self.predictions\n",
    "\n",
    "    def evaluate_metrics(self, y):\n",
    "\n",
    "        if isinstance(y, list):\n",
    "            list(y)\n",
    "        truth_values = [p == t for p, t in zip(self.predictions, y)]\n",
    "        self.misclassified_idx_ = [idx for idx,\n",
    "                                   t in enumerate(truth_values) if not t]\n",
    "        self.metrics = {'accuracy': sum(truth_values)/len(y)}\n",
    "        self.prediction_df = pd.DataFrame(self.log_probabilities).T\n",
    "        self.prediction_df.columns = self.labels\n",
    "        self.prediction_df['text'] = self.test_features_\n",
    "        return self.metrics\n",
    "\n",
    "    def get_parameter_value(self, word):\n",
    "        if word in mnb.vocab:\n",
    "            return [mnb.parameters['param_'+c][word] for c in mnb.labels]\n",
    "        else:\n",
    "            return np.nan\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "id": "0a15b379",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 12.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "mnb = MultinomialNaiveBayesCustom(alpha=1)\n",
    "mnb.fit(train_set, 'TEXT', 'label')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "f9e3540a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'p_spam': 0.13417096701817366,\n",
       " 'p_ham': 0.8658290329818263,\n",
       " 'n_spam': 15148,\n",
       " 'n_ham': 57196,\n",
       " 'n_vocab': 7670}"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnb.stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "b5a732d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TEXT</th>\n",
       "      <th>label</th>\n",
       "      <th>cricket</th>\n",
       "      <th>sing</th>\n",
       "      <th>enters</th>\n",
       "      <th>changes</th>\n",
       "      <th>45pm</th>\n",
       "      <th>collected</th>\n",
       "      <th>manege</th>\n",
       "      <th>quizzes</th>\n",
       "      <th>...</th>\n",
       "      <th>lit</th>\n",
       "      <th>shindig</th>\n",
       "      <th>thepub</th>\n",
       "      <th>forgets</th>\n",
       "      <th>prometazine</th>\n",
       "      <th>09066362220</th>\n",
       "      <th>sportsx</th>\n",
       "      <th>scool</th>\n",
       "      <th>toss</th>\n",
       "      <th>vijaykanth</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>you have an important customer service announc...</td>\n",
       "      <td>spam</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>i m reaching home in 5 min</td>\n",
       "      <td>ham</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>it s reassuring  in this crazy world</td>\n",
       "      <td>ham</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>are you staying in town</td>\n",
       "      <td>ham</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>you will go to walmart  i ll stay</td>\n",
       "      <td>ham</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 7672 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                TEXT label  cricket  sing  \\\n",
       "0  you have an important customer service announc...  spam        0     0   \n",
       "1                        i m reaching home in 5 min    ham        0     0   \n",
       "2              it s reassuring  in this crazy world    ham        0     0   \n",
       "3                          are you staying in town     ham        0     0   \n",
       "4                 you will go to walmart  i ll stay    ham        0     0   \n",
       "\n",
       "   enters  changes  45pm  collected  manege  quizzes  ...  lit  shindig  \\\n",
       "0       0        0     0          0       0        0  ...    0        0   \n",
       "1       0        0     0          0       0        0  ...    0        0   \n",
       "2       0        0     0          0       0        0  ...    0        0   \n",
       "3       0        0     0          0       0        0  ...    0        0   \n",
       "4       0        0     0          0       0        0  ...    0        0   \n",
       "\n",
       "   thepub  forgets  prometazine  09066362220  sportsx  scool  toss  vijaykanth  \n",
       "0       0        0            0            0        0      0     0           0  \n",
       "1       0        0            0            0        0      0     0           0  \n",
       "2       0        0            0            0        0      0     0           0  \n",
       "3       0        0            0            0        0      0     0           0  \n",
       "4       0        0            0            0        0      0     0           0  \n",
       "\n",
       "[5 rows x 7672 columns]"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnb.data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "id": "bbe35871",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['spam', 'ham'], dtype=object)"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnb.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "id": "cc0ffc51",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 2.25 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "preds = mnb.predict(test_set.sms.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "id": "9c3adb76",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 30.9 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "metrics = mnb.evaluate_metrics(test_set.label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "id": "5e4ac02c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.9856502242152466}"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "id": "5f397b9f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sms</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>233</th>\n",
       "      <td>U're welcome... Caught u using broken english ...</td>\n",
       "      <td>ham</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>290</th>\n",
       "      <td>Hi, Mobile no.  &amp;lt;#&amp;gt;  has added you in th...</td>\n",
       "      <td>ham</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>353</th>\n",
       "      <td>Hi, Mobile no.  &amp;lt;#&amp;gt;  has added you in th...</td>\n",
       "      <td>ham</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>379</th>\n",
       "      <td>Are you free now?can i call now?</td>\n",
       "      <td>ham</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>394</th>\n",
       "      <td>Text me when you get off, don't call, my phone...</td>\n",
       "      <td>ham</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>417</th>\n",
       "      <td>Would you like to see my XXX pics they are so ...</td>\n",
       "      <td>spam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>575</th>\n",
       "      <td>Missed call alert. These numbers called but le...</td>\n",
       "      <td>spam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>631</th>\n",
       "      <td>Cheers for the message Zogtorius. Ive been st...</td>\n",
       "      <td>ham</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>673</th>\n",
       "      <td>Total video converter free download type this ...</td>\n",
       "      <td>ham</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>733</th>\n",
       "      <td>and  picking them up from various points</td>\n",
       "      <td>ham</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>813</th>\n",
       "      <td>Santa Calling! Would your little ones like a c...</td>\n",
       "      <td>spam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>887</th>\n",
       "      <td>Madam,regret disturbance.might receive a refer...</td>\n",
       "      <td>ham</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>917</th>\n",
       "      <td>Hello darling how are you today? I would love ...</td>\n",
       "      <td>spam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>957</th>\n",
       "      <td>Did u receive my msg?</td>\n",
       "      <td>ham</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1022</th>\n",
       "      <td>Your daily text from me – a favour this time</td>\n",
       "      <td>ham</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1102</th>\n",
       "      <td>Your next amazing xxx PICSFREE1 video will be ...</td>\n",
       "      <td>spam</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    sms label\n",
       "233   U're welcome... Caught u using broken english ...   ham\n",
       "290   Hi, Mobile no.  &lt;#&gt;  has added you in th...   ham\n",
       "353   Hi, Mobile no.  &lt;#&gt;  has added you in th...   ham\n",
       "379                    Are you free now?can i call now?   ham\n",
       "394   Text me when you get off, don't call, my phone...   ham\n",
       "417   Would you like to see my XXX pics they are so ...  spam\n",
       "575   Missed call alert. These numbers called but le...  spam\n",
       "631   Cheers for the message Zogtorius. Ive been st...   ham\n",
       "673   Total video converter free download type this ...   ham\n",
       "733            and  picking them up from various points   ham\n",
       "813   Santa Calling! Would your little ones like a c...  spam\n",
       "887   Madam,regret disturbance.might receive a refer...   ham\n",
       "917   Hello darling how are you today? I would love ...  spam\n",
       "957                               Did u receive my msg?   ham\n",
       "1022       Your daily text from me – a favour this time   ham\n",
       "1102  Your next amazing xxx PICSFREE1 video will be ...  spam"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_set[test_set.index.isin(mnb.misclassified_idx_)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "id": "9b09868c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('call', 0.012533964414059077),\n",
       " ('free', 0.008239109474975897),\n",
       " ('2', 0.0071873082654045056),\n",
       " ('u', 0.0060478569550354985),\n",
       " ('txt', 0.005390481199053379),\n",
       " ('ur', 0.0050398807958629156),\n",
       " ('stop', 0.004470155140678412),\n",
       " ('mobile', 0.004470155140678412),\n",
       " ('4', 0.004470155140678412),\n",
       " ('text', 0.004207204838285564),\n",
       " ('claim', 0.004119554737487948),\n",
       " ('1', 0.003944254535892716),\n",
       " ('reply', 0.0038127793846962923),\n",
       " ('www', 0.0036374791831010607),\n",
       " ('prize', 0.003286878779910597),\n",
       " ('get', 0.0028048032255237093),\n",
       " ('cash', 0.0026733280743272856),\n",
       " ('send', 0.0025856779735296694),\n",
       " ('uk', 0.0024980278727320536),\n",
       " ('urgent', 0.00236655272153563),\n",
       " ('new', 0.00236655272153563),\n",
       " ('nokia', 0.002322727671136822),\n",
       " ('150p', 0.0022789026207380137),\n",
       " ('contact', 0.00214742746954159),\n",
       " ('com', 0.00214742746954159),\n",
       " ('please', 0.002103602419142782),\n",
       " ('win', 0.002059777368743974),\n",
       " ('msg', 0.002059777368743974),\n",
       " ('50', 0.002059777368743974),\n",
       " ('tone', 0.002059777368743974),\n",
       " ('co', 0.0020159523183451663),\n",
       " ('c', 0.001972127267946358),\n",
       " ('service', 0.00192830221754755),\n",
       " ('week', 0.0018844771671487422),\n",
       " ('guaranteed', 0.0018844771671487422),\n",
       " ('500', 0.0018406521167499343),\n",
       " ('16', 0.0018406521167499343),\n",
       " ('3', 0.0017968270663511264),\n",
       " ('per', 0.0017530020159523183),\n",
       " ('phone', 0.0017091769655535104),\n",
       " ('18', 0.0017091769655535104),\n",
       " ('customer', 0.0017091769655535104),\n",
       " ('100', 0.0014900517135594707),\n",
       " ('awarded', 0.0014900517135594707),\n",
       " ('cs', 0.0014900517135594707),\n",
       " ('1000', 0.0014900517135594707),\n",
       " ('draw', 0.0014900517135594707),\n",
       " ('sms', 0.0014900517135594707),\n",
       " ('mins', 0.0014024016127618547),\n",
       " ('message', 0.0013585765623630468)]"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(mnb.parameters['param_spam'].items(), key=lambda k:k[1], reverse=True)[:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "id": "8d415916",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('u', 0.012271451916258133),\n",
       " ('2', 0.003946597601208646),\n",
       " ('get', 0.003653686060493941),\n",
       " ('gt', 0.0036228532667344987),\n",
       " ('lt', 0.0035920204729750563),\n",
       " ('ok', 0.0033453581228995158),\n",
       " ('go', 0.0033145253291400733),\n",
       " ('ur', 0.0031603613603428606),\n",
       " ('good', 0.0029753645977862054),\n",
       " ('got', 0.002944531804026763),\n",
       " ('know', 0.002944531804026763),\n",
       " ('come', 0.002836617025868714),\n",
       " ('call', 0.0028057842321092714),\n",
       " ('like', 0.002605371072672895),\n",
       " ('day', 0.002605371072672895),\n",
       " ('time', 0.0024820398976351248),\n",
       " ('love', 0.0024512071038756823),\n",
       " ('4', 0.002327875928837912),\n",
       " ('going', 0.002142879166281257),\n",
       " ('one', 0.0020966299756420928),\n",
       " ('ü', 0.002050380785002929),\n",
       " ('want', 0.001988715197484044),\n",
       " ('home', 0.001988715197484044),\n",
       " ('lor', 0.001973298800604323),\n",
       " ('sorry', 0.001973298800604323),\n",
       " ('da', 0.0019424660068448802),\n",
       " ('need', 0.001927049609965159),\n",
       " ('k', 0.0018962168162057163),\n",
       " ('still', 0.0018808004193259951),\n",
       " ('see', 0.0018037184349273888),\n",
       " ('dont', 0.0017574692442882249),\n",
       " ('r', 0.0017266364505287824),\n",
       " ('today', 0.0016649708630098973),\n",
       " ('think', 0.0016649708630098973),\n",
       " ('later', 0.0016649708630098973),\n",
       " ('back', 0.0016187216723707336),\n",
       " ('take', 0.0016033052754910123),\n",
       " ('pls', 0.0015724724817315697),\n",
       " ('send', 0.0015570560848518484),\n",
       " ('hi', 0.0015416396879721272),\n",
       " ('tell', 0.001526223291092406),\n",
       " ('n', 0.001526223291092406),\n",
       " ('well', 0.0015108068942126845),\n",
       " ('oh', 0.001479974100453242),\n",
       " ('night', 0.0014645577035735209),\n",
       " ('dear', 0.0014028921160546357),\n",
       " ('hope', 0.0014028921160546357),\n",
       " ('much', 0.0013874757191749145),\n",
       " ('wat', 0.0013720593222951933),\n",
       " ('great', 0.0013566429254154718)]"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(mnb.parameters['param_ham'].items(), key=lambda k:k[1], reverse=True)[:50]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ed46313",
   "metadata": {},
   "source": [
    "### Try on a new dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "id": "1afb520c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "id": "8dd6789a",
   "metadata": {},
   "outputs": [],
   "source": [
    "imdb_data = pd.read_csv('../datasets/imdb_labelled.txt', sep='\\t', header=None, names=['TEXT', 'label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "id": "058fccbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "imdb_data.label = imdb_data.label.apply(lambda x: 'positive' if x==1 else 'negative')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "id": "d2d1ab42",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['A very, very, very slow-moving, aimless movie about a distressed, drifting young man.  ',\n",
       "        'negative'],\n",
       "       ['Not sure who was more lost - the flat characters or the audience, nearly half of whom walked out.  ',\n",
       "        'negative'],\n",
       "       ['Attempting artiness with black & white and clever camera angles, the movie disappointed - became even more ridiculous - as the acting was poor and the plot and lines almost non-existent.  ',\n",
       "        'negative'],\n",
       "       ['Very little music or anything to speak of.  ', 'negative'],\n",
       "       ['The best scene in the movie was when Gerardo is trying to find a song that keeps running through his head.  ',\n",
       "        'positive']], dtype=object)"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imdb_data.head().values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "id": "c1c93a6f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "positive    0.516043\n",
       "negative    0.483957\n",
       "Name: label, dtype: float64"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imdb_data.label.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "id": "67f5feed",
   "metadata": {},
   "outputs": [],
   "source": [
    "sss = StratifiedShuffleSplit(n_splits=5, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "id": "83bb7e09",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sss.get_n_splits(imdb_data.TEXT.values, imdb_data.label.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "id": "af007586",
   "metadata": {},
   "outputs": [],
   "source": [
    "for train_index, test_index in sss.split(imdb_data.TEXT.values, imdb_data.label.values):\n",
    "#     print(\"TRAIN:\", train_index, \"TEST:\", test_index)\n",
    "    X_train, X_test = imdb_data.TEXT.values[train_index], imdb_data.TEXT.values[test_index]\n",
    "    y_train, y_test = imdb_data.label.values[train_index], imdb_data.label.values[test_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "id": "06180cfa",
   "metadata": {},
   "outputs": [],
   "source": [
    "imdb_train = pd.DataFrame(zip(X_train, y_train), columns=['TEXT', 'label'])\n",
    "imdb_test = pd.DataFrame(zip(X_test, y_test), columns=['TEXT', 'label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "id": "9e8a540a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "positive    0.516722\n",
       "negative    0.483278\n",
       "Name: label, dtype: float64"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imdb_train.label.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "74d066ee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "positive    0.513333\n",
       "negative    0.486667\n",
       "Name: label, dtype: float64"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imdb_test.label.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "id": "691c951c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((598, 2), (150, 2))"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imdb_train.shape, imdb_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "id": "49cee20d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 1.39 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "mnb = MultinomialNaiveBayesCustom(alpha=1)\n",
    "mnb.fit(imdb_train, 'TEXT', 'label')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "id": "047f22c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'p_positive': 0.5167224080267558,\n",
       " 'p_negative': 0.48327759197324416,\n",
       " 'n_positive': 5016,\n",
       " 'n_negative': 5378,\n",
       " 'n_vocab': 2345}"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnb.stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "id": "53a46c7a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TEXT</th>\n",
       "      <th>label</th>\n",
       "      <th>fest</th>\n",
       "      <th>sing</th>\n",
       "      <th>flat</th>\n",
       "      <th>bunch</th>\n",
       "      <th>loose</th>\n",
       "      <th>changes</th>\n",
       "      <th>singing</th>\n",
       "      <th>definitely</th>\n",
       "      <th>...</th>\n",
       "      <th>skilled</th>\n",
       "      <th>backed</th>\n",
       "      <th>bertolucci</th>\n",
       "      <th>awkwardly</th>\n",
       "      <th>researched</th>\n",
       "      <th>based</th>\n",
       "      <th>hayao</th>\n",
       "      <th>elderly</th>\n",
       "      <th>short</th>\n",
       "      <th>survivors</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>think of the film being like a dream</td>\n",
       "      <td>positive</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>meredith m was better than all right</td>\n",
       "      <td>positive</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>imdb ratings only go as low 1 for awful  it s ...</td>\n",
       "      <td>negative</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>it s a case of  so bad it is laughable</td>\n",
       "      <td>negative</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>unfortunately  any virtue in this film s produ...</td>\n",
       "      <td>negative</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 2347 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                TEXT     label  fest  sing  \\\n",
       "0            think of the film being like a dream     positive     0     0   \n",
       "1            meredith m was better than all right     positive     0     0   \n",
       "2  imdb ratings only go as low 1 for awful  it s ...  negative     0     0   \n",
       "3         it s a case of  so bad it is laughable      negative     0     0   \n",
       "4  unfortunately  any virtue in this film s produ...  negative     0     0   \n",
       "\n",
       "   flat  bunch  loose  changes  singing  definitely  ...  skilled  backed  \\\n",
       "0     0      0      0        0        0           0  ...        0       0   \n",
       "1     0      0      0        0        0           0  ...        0       0   \n",
       "2     0      0      0        0        0           0  ...        0       0   \n",
       "3     0      0      0        0        0           0  ...        0       0   \n",
       "4     0      0      0        0        0           0  ...        0       0   \n",
       "\n",
       "   bertolucci  awkwardly  researched  based  hayao  elderly  short  survivors  \n",
       "0           0          0           0      0      0        0      0          0  \n",
       "1           0          0           0      0      0        0      0          0  \n",
       "2           0          0           0      0      0        0      0          0  \n",
       "3           0          0           0      0      0        0      0          0  \n",
       "4           0          0           0      0      0        0      0          0  \n",
       "\n",
       "[5 rows x 2347 columns]"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnb.data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "id": "cf5a77be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['positive', 'negative'], dtype=object)"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnb.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "id": "4c04aff0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 138 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "preds = mnb.predict(imdb_test.TEXT.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "id": "0fa2cec0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 4.99 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "metrics = mnb.evaluate_metrics(imdb_test.label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "id": "5342888e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.7466666666666667}"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "id": "c62d3155",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TEXT</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Being a 90's child, I truly enjoyed this show ...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>I was left shattered from the experience of wa...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>putting the race card aside, lets look at the ...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>It just blew.</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>The writers were \"smack on\" and I think the be...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Considering the relations off screen between T...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Lovely little thriller from Hitchcock, with lo...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>It is not good.</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>There's also enough hypocrisy in this film to ...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>the movie is littered with overt racial slurs ...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>He really didn't seem to want to be hosting; h...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>Not much dialogue, not much music, the whole f...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>Hopefully, the director James Cox can turn the...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>Aside from it's terrible lead, this film has l...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>I'll put this gem up against any movie in term...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>The memories are murky but I can only say that...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>They are so easy to love, but even more easy t...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>I won't say any more - I don't like spoilers, ...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>And, FINALLY, after all that, we get to an end...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>I struggle to find anything bad to say about i...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>:) Anyway, the plot flowed smoothly and the ma...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>Predictable, but not a bad watch.</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>The aerial scenes were well-done.</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>The guy who said he's had better dialogue with...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>But this movie really got to me.</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>Nothing short of magnificent photography/cinem...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>It was horrendous.</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>20th Century Fox's ROAD HOUSE 1948) is not onl...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>With great sound effects, and impressive spec...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>Wasted two hours.</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>However, this didn't make up for the fact that...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>The structure of this film is easily the most...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130</th>\n",
       "      <td>It is shameful.</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>This film (and I use that word loosely) is an ...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>See it with your kids if you have a chance--it...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>The last 15 minutes of movie are also not bad ...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>The fish is badly made and some of its underwa...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>The film gives meaning to the phrase, \"Never i...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  TEXT     label\n",
       "10   Being a 90's child, I truly enjoyed this show ...  positive\n",
       "11   I was left shattered from the experience of wa...  negative\n",
       "12   putting the race card aside, lets look at the ...  negative\n",
       "14                                     It just blew.    negative\n",
       "18   The writers were \"smack on\" and I think the be...  positive\n",
       "22   Considering the relations off screen between T...  negative\n",
       "24   Lovely little thriller from Hitchcock, with lo...  positive\n",
       "27                                   It is not good.    negative\n",
       "31   There's also enough hypocrisy in this film to ...  negative\n",
       "34   the movie is littered with overt racial slurs ...  negative\n",
       "37   He really didn't seem to want to be hosting; h...  negative\n",
       "42   Not much dialogue, not much music, the whole f...  positive\n",
       "47   Hopefully, the director James Cox can turn the...  positive\n",
       "50   Aside from it's terrible lead, this film has l...  negative\n",
       "53   I'll put this gem up against any movie in term...  positive\n",
       "57   The memories are murky but I can only say that...  positive\n",
       "58   They are so easy to love, but even more easy t...  positive\n",
       "60   I won't say any more - I don't like spoilers, ...  positive\n",
       "61   And, FINALLY, after all that, we get to an end...  negative\n",
       "64   I struggle to find anything bad to say about i...  positive\n",
       "65   :) Anyway, the plot flowed smoothly and the ma...  positive\n",
       "68                 Predictable, but not a bad watch.    positive\n",
       "83                 The aerial scenes were well-done.    positive\n",
       "87   The guy who said he's had better dialogue with...  negative\n",
       "91                  But this movie really got to me.    positive\n",
       "97   Nothing short of magnificent photography/cinem...  positive\n",
       "105                               It was horrendous.    negative\n",
       "109  20th Century Fox's ROAD HOUSE 1948) is not onl...  negative\n",
       "111   With great sound effects, and impressive spec...  positive\n",
       "112                                Wasted two hours.    negative\n",
       "115  However, this didn't make up for the fact that...  negative\n",
       "118   The structure of this film is easily the most...  negative\n",
       "130                                  It is shameful.    negative\n",
       "132  This film (and I use that word loosely) is an ...  negative\n",
       "141  See it with your kids if you have a chance--it...  positive\n",
       "145  The last 15 minutes of movie are also not bad ...  positive\n",
       "147  The fish is badly made and some of its underwa...  negative\n",
       "148  The film gives meaning to the phrase, \"Never i...  positive"
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imdb_test[imdb_test.index.isin(mnb.misclassified_idx_)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "id": "354b8f4e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['param_positive', 'param_negative'])"
      ]
     },
     "execution_count": 212,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnb.parameters.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "id": "7b9a6bbf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('movie', 0.008675385213000129),\n",
       " ('0', 0.006992101514955328),\n",
       " ('bad', 0.006474168069403082),\n",
       " ('film', 0.006474168069403082),\n",
       " ('1', 0.006474168069403082),\n",
       " ('one', 0.004402434287194096),\n",
       " ('even', 0.003237084034701541),\n",
       " ('like', 0.002848633950537356),\n",
       " ('acting', 0.0027191505891492945),\n",
       " ('plot', 0.002460183866373171),\n",
       " ('time', 0.002201217143597048),\n",
       " ('would', 0.002201217143597048),\n",
       " ('really', 0.002071733782208986),\n",
       " ('good', 0.0018127670594328628),\n",
       " ('ever', 0.0018127670594328628),\n",
       " ('awful', 0.0018127670594328628),\n",
       " ('script', 0.0016832836980448013),\n",
       " ('made', 0.0016832836980448013),\n",
       " ('stupid', 0.0016832836980448013),\n",
       " ('films', 0.0016832836980448013),\n",
       " ('see', 0.0015538003366567395),\n",
       " ('movies', 0.0015538003366567395),\n",
       " ('show', 0.0015538003366567395),\n",
       " ('make', 0.0015538003366567395),\n",
       " ('story', 0.0015538003366567395),\n",
       " ('work', 0.0015538003366567395),\n",
       " ('characters', 0.0015538003366567395),\n",
       " ('seen', 0.0015538003366567395),\n",
       " ('nothing', 0.0015538003366567395),\n",
       " ('worst', 0.001424316975268678),\n",
       " ('much', 0.001424316975268678),\n",
       " ('real', 0.001424316975268678),\n",
       " ('way', 0.001424316975268678),\n",
       " ('well', 0.0012948336138806163),\n",
       " ('watching', 0.0012948336138806163),\n",
       " ('actors', 0.0012948336138806163),\n",
       " ('little', 0.0012948336138806163),\n",
       " ('everything', 0.0011653502524925548),\n",
       " ('thing', 0.0011653502524925548),\n",
       " ('writing', 0.0011653502524925548),\n",
       " ('great', 0.0011653502524925548),\n",
       " ('could', 0.0011653502524925548),\n",
       " ('scenes', 0.0011653502524925548),\n",
       " ('never', 0.0011653502524925548),\n",
       " ('worse', 0.0011653502524925548),\n",
       " ('every', 0.001035866891104493),\n",
       " ('10', 0.001035866891104493),\n",
       " ('look', 0.001035866891104493),\n",
       " ('sucked', 0.001035866891104493),\n",
       " ('scene', 0.001035866891104493)]"
      ]
     },
     "execution_count": 213,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(mnb.parameters['param_negative'].items(), key=lambda k:k[1], reverse=True)[:50]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ecb32ca",
   "metadata": {},
   "source": [
    "## Multinomial Naive Bayes with Dot Product Trick"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cad77ae",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}