model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classifying Tweet Emotions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import necessary modules\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pylab as plt\n",
    "#import seaborn as sns\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "#pd.set_option('display.max_colwidth', 160)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read data\n",
    "df = pd.read_pickle('data/emotions_preprocessed.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>content</th>\n",
       "      <th>polarity</th>\n",
       "      <th>mention_count</th>\n",
       "      <th>hashtag_count</th>\n",
       "      <th>char_count</th>\n",
       "      <th>has_link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sadness</td>\n",
       "      <td>layin n bed with a headache ughhhh...waitin on...</td>\n",
       "      <td>negative</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>59</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sadness</td>\n",
       "      <td>funeral ceremony...gloomy friday...</td>\n",
       "      <td>negative</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>enthusiasm</td>\n",
       "      <td>wants to hang out with friends soon!</td>\n",
       "      <td>positive</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>36</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>neutral</td>\n",
       "      <td>@user we want to trade with someone who has ho...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>77</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>worry</td>\n",
       "      <td>re-pinging @user why didn't you go to prom? bc...</td>\n",
       "      <td>negative</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>75</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    sentiment                                            content  polarity  \\\n",
       "1     sadness  layin n bed with a headache ughhhh...waitin on...  negative   \n",
       "2     sadness                funeral ceremony...gloomy friday...  negative   \n",
       "3  enthusiasm               wants to hang out with friends soon!  positive   \n",
       "4     neutral  @user we want to trade with someone who has ho...   neutral   \n",
       "5       worry  re-pinging @user why didn't you go to prom? bc...  negative   \n",
       "\n",
       "   mention_count  hashtag_count  char_count  has_link  \n",
       "1              0              0          59         0  \n",
       "2              0              0          35         0  \n",
       "3              0              0          36         0  \n",
       "4              1              0          77         0  \n",
       "5              1              0          75         0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transforming Data - Create X and y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create features and target\n",
    "X = df.drop(['sentiment', 'polarity'], axis=1)\n",
    "y = df.sentiment\n",
    "\n",
    "# Split training and testing data\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.25, random_state=57)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vectorizing Tweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### `CountVectorizer`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize count vectorizer\n",
    "count_vectorizer = CountVectorizer(stop_words='english', strip_accents='ascii',\n",
    "                                  min_df=.001)\n",
    "\n",
    "# Create count train and test variables\n",
    "X_train_bow = count_vectorizer.fit_transform(X_train.content)\n",
    "X_test_bow = count_vectorizer.transform(X_test.content)\n",
    "\n",
    "# Convert matrices into a DataFrame\n",
    "X_train_bow_df = pd.DataFrame(X_train_bow.toarray())\n",
    "X_test_bow_df = pd.DataFrame(X_test_bow.toarray())\n",
    "\n",
    "# Map the column names to vocabulary \n",
    "X_train_bow_df.columns = count_vectorizer.get_feature_names()\n",
    "X_test_bow_df.columns = count_vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(28964, 946)"
      ]
     },
     "execution_count": 241,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_bow_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['10', '100', '11', '12', '14', '15', '1st', '20', '24', '25',\n",
       "       ...\n",
       "       'yep', 'yes', 'yesterday', 'yo', 'young', 'youtube', 'yr', 'yum',\n",
       "       'yummy', 'yup'],\n",
       "      dtype='object', length=946)"
      ]
     },
     "execution_count": 242,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_bow_df.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Multinomial Naive Bayes Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/stb/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py:3940: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  errors=errors)\n"
     ]
    }
   ],
   "source": [
    "# Drop 'content'\n",
    "X_train.drop('content', axis=1, inplace=True)\n",
    "X_test.drop('content', axis=1, inplace=True)\n",
    "\n",
    "# Merge data\n",
    "X_train = X_train.reset_index(drop=True).join(X_train_bow_df)\n",
    "X_test = X_test.reset_index(drop=True).join(X_test_bow_df)\n",
    "X_test.fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Emotions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NaiveBayes Count Score:  0.3360952874158467 \n",
      "\n",
      "[[ 307    3  308  495   26   32    9   39   65    0   10    0]\n",
      " [  15    0   88   46    2   14    3    2   26    0    1    0]\n",
      " [ 119    6 1074  573   31   86   22   16  191    1   15    0]\n",
      " [ 241    9  576  955   30   55   20   35  147    2   13    0]\n",
      " [  39    1  186  177   33   45    5    5   62    0    6    0]\n",
      " [  36    2  185  126   13  359   10    0  174    0   17    0]\n",
      " [  16    1  149   96    8   21   23    2  111    0    4    0]\n",
      " [  39    0   65  112    3    8    1   61    9    0    2    0]\n",
      " [  43    2  390  203   24  165   30    3  406    0   25    0]\n",
      " [  10    0   10   21    0    2    0    3    0    1    0    0]\n",
      " [  20    0  123  101    6   25    7    6   55    0   26    0]\n",
      " [   0    0    7   17    0    2    1    0    0    0    1    0]]\n"
     ]
    }
   ],
   "source": [
    "# Create a MulitnomialNB model\n",
    "clf_nb = MultinomialNB()\n",
    "\n",
    "clf_nb.fit(X_train, y_train)\n",
    "\n",
    "# Run predict on count test data to get predictions\n",
    "count_nb_pred = clf_nb.predict(X_test)\n",
    "\n",
    "# Calculate the accuracy of predictions\n",
    "count_nb_score = clf_nb.score(X_test, y_test)\n",
    "\n",
    "print('NaiveBayes Count Score: ', count_nb_score, '\\n')\n",
    "print(confusion_matrix(y_test, count_nb_pred, labels=y.unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>anger</th>\n",
       "      <th>boredom</th>\n",
       "      <th>enthusiasm</th>\n",
       "      <th>fun</th>\n",
       "      <th>happiness</th>\n",
       "      <th>hate</th>\n",
       "      <th>love</th>\n",
       "      <th>neutral</th>\n",
       "      <th>relief</th>\n",
       "      <th>sadness</th>\n",
       "      <th>surprise</th>\n",
       "      <th>worry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>anger</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>boredom</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>enthusiasm</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>26</td>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "      <td>88</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fun</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>23</td>\n",
       "      <td>111</td>\n",
       "      <td>2</td>\n",
       "      <td>21</td>\n",
       "      <td>149</td>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>8</td>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>happiness</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>30</td>\n",
       "      <td>406</td>\n",
       "      <td>3</td>\n",
       "      <td>165</td>\n",
       "      <td>390</td>\n",
       "      <td>25</td>\n",
       "      <td>43</td>\n",
       "      <td>24</td>\n",
       "      <td>203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hate</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>61</td>\n",
       "      <td>8</td>\n",
       "      <td>65</td>\n",
       "      <td>2</td>\n",
       "      <td>39</td>\n",
       "      <td>3</td>\n",
       "      <td>112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>love</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>174</td>\n",
       "      <td>0</td>\n",
       "      <td>359</td>\n",
       "      <td>185</td>\n",
       "      <td>17</td>\n",
       "      <td>36</td>\n",
       "      <td>13</td>\n",
       "      <td>126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>neutral</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>22</td>\n",
       "      <td>191</td>\n",
       "      <td>16</td>\n",
       "      <td>86</td>\n",
       "      <td>1074</td>\n",
       "      <td>15</td>\n",
       "      <td>119</td>\n",
       "      <td>31</td>\n",
       "      <td>573</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>relief</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>55</td>\n",
       "      <td>6</td>\n",
       "      <td>25</td>\n",
       "      <td>123</td>\n",
       "      <td>26</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sadness</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>65</td>\n",
       "      <td>39</td>\n",
       "      <td>32</td>\n",
       "      <td>308</td>\n",
       "      <td>10</td>\n",
       "      <td>307</td>\n",
       "      <td>26</td>\n",
       "      <td>495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>surprise</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>62</td>\n",
       "      <td>5</td>\n",
       "      <td>45</td>\n",
       "      <td>186</td>\n",
       "      <td>6</td>\n",
       "      <td>39</td>\n",
       "      <td>33</td>\n",
       "      <td>177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>worry</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>20</td>\n",
       "      <td>147</td>\n",
       "      <td>35</td>\n",
       "      <td>55</td>\n",
       "      <td>576</td>\n",
       "      <td>13</td>\n",
       "      <td>241</td>\n",
       "      <td>30</td>\n",
       "      <td>955</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            anger  boredom  enthusiasm  fun  happiness  hate  love  neutral  \\\n",
       "anger           0        0           0    1          0     0     2        7   \n",
       "boredom         0        1           0    0          0     3     2       10   \n",
       "enthusiasm      0        0           0    3         26     2    14       88   \n",
       "fun             0        0           1   23        111     2    21      149   \n",
       "happiness       0        0           2   30        406     3   165      390   \n",
       "hate            0        0           0    1          9    61     8       65   \n",
       "love            0        0           2   10        174     0   359      185   \n",
       "neutral         0        1           6   22        191    16    86     1074   \n",
       "relief          0        0           0    7         55     6    25      123   \n",
       "sadness         0        0           3    9         65    39    32      308   \n",
       "surprise        0        0           1    5         62     5    45      186   \n",
       "worry           0        2           9   20        147    35    55      576   \n",
       "\n",
       "            relief  sadness  surprise  worry  \n",
       "anger            1        0         0     17  \n",
       "boredom          0       10         0     21  \n",
       "enthusiasm       1       15         2     46  \n",
       "fun              4       16         8     96  \n",
       "happiness       25       43        24    203  \n",
       "hate             2       39         3    112  \n",
       "love            17       36        13    126  \n",
       "neutral         15      119        31    573  \n",
       "relief          26       20         6    101  \n",
       "sadness         10      307        26    495  \n",
       "surprise         6       39        33    177  \n",
       "worry           13      241        30    955  "
      ]
     },
     "execution_count": 245,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print Confusion Matrix\n",
    "labels = np.unique(y_test)\n",
    "cm =  confusion_matrix(y_test, count_nb_pred, labels=labels)\n",
    "pd.DataFrame(cm, index=labels, columns=labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1a1a782400>"
      ]
     },
     "execution_count": 246,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEpCAYAAAB/ZvKwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3de7hdVX3u8e/LTUSBBN1QDGhQI95FjICCVUHCzRI8BYulmmJs7ClVa60Ve+pBUSxalWpPRaMEI0URUEsKCEYEqVYuCVe5HSIgRFSiCcgRRMD3/DHGgpWwL2sna67N3vP9PM9+1ppjzTl/c+7s/NaYY445hmwTERHtsNFEH0BERAxOkn5ERIsk6UdEtEiSfkREiyTpR0S0SJJ+RESLbDLRBzCapz71qZ45c+ZEH0ZExKSyfPnyX9oeGu6zx3XSnzlzJsuWLZvow4iImFQk/WSkz9K8ExHRIkn6EREtkqQfEdEiSfoRES2SpB8R0SJJ+hERLZKkHxHRIkn6EREt8rh+OGs0M48+Z722u+34g/p8JBERk0dq+hERLZKkHxHRIpO2eWfQ0pwUEVNBavoRES2SpB8R0SJJ+hERLdJT0pf0bknXSfqRpK9K2lzSTpIulXSzpK9J2qyu+4S6vKJ+PrNrP++v5TdJ2q+ZU4qIiJGMmfQlzQDeCcy2/UJgY+Bw4GPACbZnAWuA+XWT+cAa288GTqjrIen5dbsXAPsDn5W0cX9PJyIiRtNr884mwBMlbQJsAfwM2Bs4s36+GDikvp9bl6mf7yNJtfw02w/YvhVYAey24acQERG9GjPp2/4p8AngdkqyvwdYDtxt+6G62kpgRn0/A7ijbvtQXf8p3eXDbBMREQPQS/POdEotfSfgacCTgAOGWdWdTUb4bKTydeMtkLRM0rJVq1aNdXgRETEOvTTvvA641fYq2w8C3wBeCUyrzT0AOwB31vcrgR0B6udbA6u7y4fZ5hG2F9qebXv20NCwk7lHRMR66iXp3w7sIWmL2ja/D3A9cCFwaF1nHnBWfb+kLlM//65t1/LDa++enYBZwGX9OY2IiOjFmMMw2L5U0pnAFcBDwJXAQuAc4DRJH6llJ9VNTgJOkbSCUsM/vO7nOkmnU74wHgKOsv1wn88nIiJG0dPYO7aPAY5Zp/gWhul9Y/u3wGEj7Oc44LhxHmNERPRJnsiNiGiRJP2IiBZJ0o+IaJEk/YiIFknSj4hokST9iIgWSdKPiGiRJP2IiBZJ0o+IaJEk/YiIFknSj4hokST9iIgWSdKPiGiRJP2IiBZJ0o+IaJEk/YiIFullYvSdJV3V9fNrSX8jaRtJSyXdXF+n1/Ul6TOSVki6RtKuXfuaV9e/WdK8kaNGREQTxkz6tm+yvYvtXYCXAfcB3wSOBi6wPQu4oC4DHECZ/3YWsAA4EUDSNpTZt3anzLh1TOeLIiIiBmO8zTv7AD+2/RNgLrC4li8GDqnv5wJfdnEJME3S9sB+wFLbq22vAZYC+2/wGURERM/Gm/QPB75a329n+2cA9XXbWj4DuKNrm5W1bKTyiIgYkJ6TvqTNgIOBM8ZadZgyj1K+bpwFkpZJWrZq1apeDy8iInownpr+AcAVtn9Rl39Rm22or3fV8pXAjl3b7QDcOUr5WmwvtD3b9uyhoaFxHF5ERIxlPEn/TTzatAOwBOj0wJkHnNVV/pbai2cP4J7a/HM+MEfS9HoDd04ti4iIAdmkl5UkbQHsC7y9q/h44HRJ84HbgcNq+bnAgcAKSk+fIwFsr5b0YeDyut6xtldv8BlERETPekr6tu8DnrJO2a8ovXnWXdfAUSPsZxGwaPyHGRER/ZAnciMiWiRJPyKiRZL0IyJaJEk/IqJFkvQjIlokST8iokWS9CMiWiRJPyKiRZL0IyJaJEk/IqJFkvQjIlokST8iokWS9CMiWiRJPyKiRZL0IyJapKekL2mapDMl3SjpBkmvkLSNpKWSbq6v0+u6kvQZSSskXSNp1679zKvr3yxp3sgRIyKiCb3W9D8NnGf7ucBLgBuAo4ELbM8CLqjLUObSnVV/FgAnAkjaBjgG2B3YDTim80URERGDMWbSl7QV8IfASQC2f2f7bmAusLiuthg4pL6fC3zZxSXAtDpx+n7AUturba8BlgL79/VsIiJiVL3U9J8JrAJOlnSlpC9KehKwXZ3wnPq6bV1/BnBH1/Yra9lI5RERMSC9JP1NgF2BE22/FPgNjzblDEfDlHmU8rU3lhZIWiZp2apVq3o4vIiI6FUvSX8lsNL2pXX5TMqXwC9qsw319a6u9Xfs2n4H4M5Rytdie6Ht2bZnDw0NjedcIiJiDGMmfds/B+6QtHMt2ge4HlgCdHrgzAPOqu+XAG+pvXj2AO6pzT/nA3MkTa83cOfUsoiIGJBNelzvHcCpkjYDbgGOpHxhnC5pPnA7cFhd91zgQGAFcF9dF9urJX0YuLyud6zt1X05i4iI6ElPSd/2VcDsYT7aZ5h1DRw1wn4WAYvGc4AREdE/eSI3IqJFkvQjIlokST8iokWS9CMiWiRJPyKiRZL0IyJaJEk/IqJFkvQjIlokST8iokWS9CMiWiRJPyKiRZL0IyJaJEk/IqJFkvQjIlokST8iokWS9CMiWqSnSVQk3QbcCzwMPGR7tqRtgK8BM4HbgDfaXiNJwKcps2fdB/y57SvqfuYB/1h3+xHbi/t3KlPLzKPPWa/tbjv+oD4fSURMJb1OlwjwWtu/7Fo+GrjA9vGSjq7L7wMOAGbVn92BE4Hd65fEMZQZuAwsl7TE9po+nEdsoHzJRLTDhjTvzAU6NfXFwCFd5V92cQkwTdL2wH7AUtura6JfCuy/AfEjImKcek36Br4tabmkBbVsO9s/A6iv29byGcAdXduurGUjlUdExID02ryzp+07JW0LLJV04yjrapgyj1K+9sblS2UBwNOf/vQeDy8iInrRU03f9p319S7gm8BuwC9qsw319a66+kpgx67NdwDuHKV83VgLbc+2PXtoaGh8ZxMREaMaM+lLepKkLTvvgTnAj4AlwLy62jzgrPp+CfAWFXsA99Tmn/OBOZKmS5pe93N+X88mIiJG1UvzznbAN0tPTDYBvmL7PEmXA6dLmg/cDhxW1z+X0l1zBaXL5pEAtldL+jBweV3vWNur+3YmERExpjGTvu1bgJcMU/4rYJ9hyg0cNcK+FgGLxn+YERHRD3kiNyKiRZL0IyJaJEk/IqJFkvQjIlokST8iokWS9CMiWiRJPyKiRZL0IyJaJEk/IqJFkvQjIlokST8iokWS9CMiWiRJPyKiRZL0IyJaJEk/IqJFek76kjaWdKWks+vyTpIulXSzpK9J2qyWP6Eur6ifz+zax/tr+U2S9uv3yURExOjGU9N/F3BD1/LHgBNszwLWAPNr+Xxgje1nAyfU9ZD0fOBw4AXA/sBnJW28YYcfERHj0VPSl7QDcBDwxbosYG/gzLrKYuCQ+n5uXaZ+vk9dfy5wmu0HbN9KmU5xt36cRERE9KbXmv6/AH8P/L4uPwW42/ZDdXklMKO+nwHcAVA/v6eu/0j5MNtERMQAjJn0Jb0euMv28u7iYVb1GJ+Ntk13vAWSlklatmrVqrEOLyIixqGXmv6ewMGSbgNOozTr/AswTVJnYvUdgDvr+5XAjgD1862B1d3lw2zzCNsLbc+2PXtoaGjcJxQRESMbM+nbfr/tHWzPpNyI/a7tI4ALgUPravOAs+r7JXWZ+vl3bbuWH1579+wEzAIu69uZRETEmDYZe5URvQ84TdJHgCuBk2r5ScApklZQaviHA9i+TtLpwPXAQ8BRth/egPgRETFO40r6ti8CLqrvb2GY3je2fwscNsL2xwHHjfcgIyKiP/JEbkREi2xI807Eept59Dnrtd1txx/U5yOJaJfU9CMiWiRJPyKiRZL0IyJaJEk/IqJFkvQjIlokST8iokWS9CMiWiRJPyKiRZL0IyJaJEk/IqJFkvQjIlokST8iokWS9CMiWiRJPyKiRXqZGH1zSZdJulrSdZI+VMt3knSppJslfU3SZrX8CXV5Rf18Zte+3l/Lb5K0X1MnFRERw+ulpv8AsLftlwC7APtL2gP4GHCC7VnAGmB+XX8+sMb2s4ET6npIej5l6sQXAPsDn5W0cT9PJiIiRtfLxOi2/f/q4qb1x8DewJm1fDFwSH0/ty5TP99Hkmr5abYfsH0rsIJhpluMiIjm9NSmL2ljSVcBdwFLgR8Dd9t+qK6yEphR388A7gCon98DPKW7fJhtIiJiAHpK+rYftr0LsAOldv684Varrxrhs5HK1yJpgaRlkpatWrWql8OLiIgejav3ju27gYuAPYBpkjpz7O4A3FnfrwR2BKifbw2s7i4fZpvuGAttz7Y9e2hoaDyHFxERY+il986QpGn1/ROB1wE3ABcCh9bV5gFn1fdL6jL18+/adi0/vPbu2QmYBVzWrxOJiIixbTL2KmwPLK49bTYCTrd9tqTrgdMkfQS4Ejiprn8ScIqkFZQa/uEAtq+TdDpwPfAQcJTth/t7OhERMZoxk77ta4CXDlN+C8P0vrH9W+CwEfZ1HHDc+A8zIiL6IU/kRkS0SJJ+RESLJOlHRLRIkn5ERIsk6UdEtEiSfkREiyTpR0S0SJJ+RESLJOlHRLRIkn5ERIsk6UdEtEiSfkREiyTpR0S0SJJ+RESLJOlHRLRIL5OoREx6M48+Z722u+34g/p8JBETq5fpEneUdKGkGyRdJ+ldtXwbSUsl3Vxfp9dySfqMpBWSrpG0a9e+5tX1b5Y0b6SYERHRjF6adx4C3mP7eZQJ0Y+S9HzgaOAC27OAC+oywAGU+W9nAQuAE6F8SQDHALtTZtw6pvNFERERgzFm0rf9M9tX1Pf3UiZFnwHMBRbX1RYDh9T3c4Evu7gEmCZpe2A/YKnt1bbXAEuB/ft6NhERMapx3ciVNJMyX+6lwHa2fwbliwHYtq42A7ija7OVtWyk8nVjLJC0TNKyVatWjefwIiJiDD0nfUlPBr4O/I3tX4+26jBlHqV87QJ7oe3ZtmcPDQ31engREdGDnpK+pE0pCf9U29+oxb+ozTbU17tq+Upgx67NdwDuHKU8IiIGpJfeOwJOAm6w/amuj5YAnR4484CzusrfUnvx7AHcU5t/zgfmSJpeb+DOqWURETEgvfTT3xN4M3CtpKtq2T8AxwOnS5oP3A4cVj87FzgQWAHcBxwJYHu1pA8Dl9f1jrW9ui9nERERPRkz6dv+PsO3xwPsM8z6Bo4aYV+LgEXjOcCIiOifPJEb0YA8ARyPVxl7JyKiRZL0IyJaJEk/IqJFkvQjIlokST8iokXSeydiCkhvoehVavoRES2SpB8R0SJJ+hERLZKkHxHRIrmRGxHjtj43jnPT+PEhNf2IiBZJ0o+IaJEk/YiIFull5qxFku6S9KOusm0kLZV0c32dXssl6TOSVki6RtKuXdvMq+vfLGnecLEiIqJZvdT0vwTsv07Z0cAFtmcBF9RlgAOAWfVnAXAilC8J4Bhgd2A34JjOF0VERAzOmEnf9sXAutMazgUW1/eLgUO6yr/s4hJgWp00fT9gqe3VttcAS3nsF0lERDRsfdv0t6uTnVNft63lM4A7utZbWctGKo+IiAHq943c4ebS9Sjlj92BtEDSMknLVq1a1deDi4hou/VN+r+ozTbU17tq+Upgx671dgDuHKX8MWwvtD3b9uyhoaH1PLyIiBjO+ib9JUCnB8484Kyu8rfUXjx7APfU5p/zgTmSptcbuHNqWUREDNCYwzBI+irwGuCpklZSeuEcD5wuaT5wO3BYXf1c4EBgBXAfcCSA7dWSPgxcXtc71va6N4cjIqJhYyZ9228a4aN9hlnXwFEj7GcRsGhcRxcREX2VJ3IjIloko2xGxONapoLsr9T0IyJaJEk/IqJFkvQjIlokST8iokWS9CMiWiRJPyKiRZL0IyJaJEk/IqJFkvQjIlokST8iokWS9CMiWiRJPyKiRZL0IyJaJKNsRkR0meqjeg68pi9pf0k3SVoh6ehBx4+IaLOB1vQlbQz8G7AvZbL0yyUtsX39II8jIuLxYtBXFoOu6e8GrLB9i+3fAacBcwd8DBERrTXopD8DuKNreWUti4iIAVCZy3xAwaTDgP1sv60uvxnYzfY7utZZACyoizsDN61HqKcCv9zAw028xEu8x3esxBvZM2wPDffBoHvvrAR27FreAbizewXbC4GFGxJE0jLbszdkH4mXeIn3+I6VeOtn0M07lwOzJO0kaTPgcGDJgI8hIqK1BlrTt/2QpL8Gzgc2BhbZvm6QxxAR0WYDfzjL9rnAuQ2H2aDmocRLvMSbFLESbz0M9EZuRERMrIy9ExHRIkn6EREtkqQfY5I0XdKLJ/o4YnSSLqivH5voY4n1o2LHsddcfxllcz1JWgacDHzF9poBxHsScL/t30t6DvBc4Fu2H2wo3kXAwZS/kauAVZK+Z/tvm4hXYz4DmGX7O5KeCGxi+94G4mwHfBR4mu0DJD0feIXtk/odq8Z7AvDHwEy6/s/ZPrbPobaX9GrgYEmnAer+0PYVfY4HQP17PBHYzvYLawXhYNsfaSjexsBBPPb3+amG4u0EvGOYeAf3O5ZtS/oP4GX93nfHpL+RK+leYLiTEOV3uFVDcZ8NHAn8CdD5Avi2G/qFSloOvAqYDlxSY95n+4iG4l1p+6WS3gbsaPsYSdfYbqTGL+kvKE9ib2P7WZJmAZ+zvU8Dsb5F+ff6X7ZfImkT4ErbL+p3rBrvPOAeYDnwcKfc9if7HOdQYD6wF+Xvo5tt793PeF1xvwe8F/i87ZfWsh/ZfmFD8c4FfgtcC/y+U277Qw3Fuxo4aZh432so3r8BX7J9eRP7x3Z+NuCH0kR2MPBTyrhCH6Ikrn7HuaK+vgP4+/r+ygbP61pge+DbwMtr2TUNxrsK2Kz7nIBrG4p1+bq/P+CqBs/tR03te4R4HxhwvEH/Phv7Oxwh3qUDjnc9pXLwY+Ca+n+xb+c85Zp3JG0LbN5Ztn17g7FeTKntHwh8HTiVUsv6LrBL/8PpFcARlNocNNs8dyzlIbrv275c0jOBmxuM94Dt30mlRaLWvpu6DP2NpKd09i9pD0pNvCn/LelFtq9tMEa34yT9GfBM28dKejrwB7YvayjeLyU9i0d/n4cCP2soFsC3JM2x/e0GY3T7tKRjKBWgBzqFbqi5DDigof0CU6B5p0PSwcAngacBdwHPAG6w/YKG4i0H7qZc9n3d9gNdn33D9v/oc7xXA+8BfmD7YzUJ/43td/YzzkSR9HHK7/MtlKuZvwKut/2/Goi1K/CvwAuBHwFDwKG2r+l3rBrveuDZwK2UpNFpemyqqexESjPE3rafJ2k6penx5Q3FeyblIaJXAmso53mE7Z80FO8NwL9TrrIfpPmm3H8C3kypeXead+yGmstqzL0o97dOljQEPNn2rX3Z9xRK+lcDewPfcWmLfi3wJtsLxth0fWJtBBxt+6P93vc44j/Z9q8bjPFx4CPA/cB5wEsoXzL/3lC8jShXMHMo/4nPB77ohv5A65XEzjXWTW7ohniN9YzhyhtMilfY3rVzX6aWXW37JQ3F28n2rbWzwUa27+2UNRTvFuAQSvNf4wlM0o3Ai13mAGlcvaqYDexs+zmSngacYXvPfux/KnXZfND2r4CNJG1k+0L638QCgO3fA/s3se+RSPqKpK3qf6zrgZskvbfBkHPql8rrKaOjPodys64pc4Ev2z7M9qG2v9Bgwj8MeKLLuE+HAF+rtf+meISfpjxYe7h0mluG6LoB2YCvA9j+jR/tbXVmg/FuptwnGVSN9Wpg2oBiAbyBcp/wNwC27wS27NfOp1Kb/t2SngxcDJwq6S7goQbjLZX0d8DXqP84ALZXNxTv+bZ/LekIythF76P0BvnnhuJtWl8PBL5qe3Wnvb0hBwP/Iuliyoxq59tu6t/vA7bPqJfQ+wGfoHQ53L2heOdQErAo95t2oswT0UjTI/AZ4JvAtpKOAw4F/rHfQSQ9l3IOW0vqbs7ciq77ag34GXBR7YXV3cbeSJdNYDvgRkmXrxOv7102q9/ZtqTOl/aT+rnzqZT051KaIt5Nudm5NeVmZFPeWl+P6ioz8MyG4m0qaVNKzfT/2H6w80fRkP+sl7X3A39Va4u/bSqY7SPr+R0A/CnwWUlLXSfc6bNOt8mDgBNtnyXpgw3EAcDrdAWtVxVvbzDeqfWe0z6UL5pDbN/QQKidKVeC04A/6iq/F/iLBuJ13Fp/Nqs/TTtmADG6nS7p88C02pX5rcAX+rXzKdGmXy9lz7f9ugHF24jyMM8PBhGvxnwnpXZ/NSVZPR34d9uvajDmdODXth+utY0tbf+8qXg15qaUprMjgVd5hNl/NjDG2ZQutq+jPARzP3BZU23eIxzDFbb72qQkaat6NbjNcJ83dRUq6RW2f9jEvnuI3fj9rRpnO6BzI/wy23c1HG9fuu5v2V7at31PhaQPIGkJ8GbbTXa96473Q9uvGESsUY5hk6aaQCRtAfwt8HTbC+rDUjvbPruhePtTJtV5LXARpdns202cXz23/Sk3Am+WtD3woqa6AErqfop5I8oXzTa29+tznLNtv17SrTzanPTIq+1GrkIlbU65Cf8C1u4u/dYRN9qweF8B/pJyxbacclX/KduNNHVKeiOlGfUiyu/yVcB7bTd536IxUynpnw7sASxl7Tb2Rro0SvoQ5cGJbwyoB8Gghw74GuU/1FtcHq1/IvBD243cHFcZNuA0ytASD4y1/nrGGGhNWNIptt8s6W7ghFr8EHAbpZtvY81lgyTpDOBGSrPcsZTm1Rtsv6uheFfZ3qXe33oZ9f5Wg11grwb27dTua1PndxrsDTXcKAP3UJ6yfo/tWzZk/1OpTf+c+tOtyWT8t8CTgIcl3U/DfYWBL1GHDqjL/5dSG24k6QPPsv0nkt4EYPt+NXgn1/bh9Ytt3xqmiUvor1DaoJfzaA34kUOg//djXla7a95OeS6g2xb0+R7JWD2QGnyY6Nm2D5M01/biWhM/v6FYMPj7Wxut87f4K5rt+fgpytzhX6H8jR4O/AHl5v8i4DUbsvOplPSn2f50d4GkRmoaALb71oWqR0+1fbqk99f4D0l6eKyNNsDvau2+04PgWXT1XOi32o3yEzx6Cf2vkvp6CV2bPgS82g0+qd3lc5RnHHZi7bFwOs0u/f6SGW0sH1OeY2lC5xmHuyW9EPg5ZXCypnyecrV0NXBx/WJtsk3/PEnnA1+ty39Cs7P/7W+7uyfZQkmXuDxd/Q8bvHcPcEyJJn+oY9OsU9bY2DR1/wdTEtUngNc3HOsi4Ck8OgbPHsD3Goy3L/A9YBVleInbgNc0GO9qYNuu5SHg6oZiLW/y32qYeCcOMt6gf4C3UQYC/EPgFsoT8W8f8DFs0vD+/5hSAz8BeEPDsX4IvJFyNbFRfX9J/WyDxzSa9G36tfnhTylj3vxX10dbAg+7oR49ko6n3M0/tRa9iZJMjm4o3kCHDqgxn0L5chHlj+6XDca61l1dG2uvjKvdwMiXanoUwwk2ATfhu4eO7jzfYfd/6OjumAfx2BvHTXbRHhiVYS0+DbyCcoV2CaUr+k+Bl9n+/gbtfwok/WdQLp//CehOuPdSRqZrqnfLNcAuLk/ndrqNXumGbibVGAMbOqDGm0EZw6h7DPGLG4r1z8CLWfsS+hrb72sg1vWUJ4x/Qrnp3+hYOIM2ATfhBzJ0dFe8z1HuibwW+CLl4bPLbM8fdcPxxxlp2HYA3Nz9u0ZN+qQ/UWrSf41rj4/aI+SihpP+K3nsRA5fbijWxyiJ9zrWHmSqqacQkfTHwJ6UJHyx7W82FGegY+EMmqRltmdrcGPvNDZ2/gjxrrH94q7XJ1N60c1pKN6xlPsUp1D+No+gPLPy8YbiDVEebpvJ2v/X+9IFdsrcyF3nW3kzymXmbxr8Nv4ocIXKDFOitGe+v6FYSDoFeBZl3PlObcpAI0mf0jNiZzfUfXI4tr9OHcel4Tg/qc1le1F+hz9wcz1bJsJAb8Iz+KGj76+v96kMRvYrytV+U/bz2jdWT5R0KdBI0gfOojRVf4euK6d+mTJJ3+v0ppF0CLBbgyEPonSfWkPpkvc+N/u06mzK+DuDujS7hfLF2WjSH+USurEusJL+N3AY8I1adLKkM9zQ9H6DVHsndXoN7SjpVMrV0583EOtayr/dJsCRKqNfNj50NHC2pGmUB6auqMfwxYZiQemWfQTlORJT7t812XNuiyaaNTumdPNO7ea0R0P73ptSU3wVpevdVZQmiU+PuuH6xzsDeKftJien6I73dcpwyhew9iBTk378fkk3AC91fTiq1oqvsP28iT2y/lAZd2cODd+EH6mZrGMQzWX1JvLmbvBJfEkzKTdW96ReGVKGGb+toXgfAf7bdiPdQqdMTV9rj/K3EaVm3Ng3mu3vqswN+nLKDaW/pPQmaCTpA08Frpd0GYMZ6W9J/ZmKbqP0+ug8HPUEygQZU8UllFmz1n1Ysa8GfQ9E0t71/91jJiiShO1vDLfdhqrJfW4T+x7Bu4B/kPQADUwSM2WSPmuP8td51L2xfyhJF1CeyP0hpf3t5W52EKYPNrjvx7C9eJDxBuwB4DpJSykVg32B70v6DEyJq5nXAm+XNNV6J72aMhXpHw3zmXm0ua6vJD2HMvT2drU31IuBg5tqDrS9Ze0YMosGhqie0s07TZJ0AmXcjwcol3sXU7rF3T/qho9zXe20w5oCiQNJ80b7fLJ/4U313kmDVq/o3wt8vqs3VGM9liS9jVLb34HSbLwHpblnn37sf8rU9Cfg2/jdNe6TKcMAn0wZH+MJ/Ywj6fu29xrmhmdTNzpfX1878wScUl+PAO7rc6yBq89T7Gv7zyb6WJoy1ZN7HV7lZMqzOF8AdqVMX9rUROlb2L5snaGnmpyg6V2UZuNLbL9WZbKaD/Vr51NpusQvULpMPghQn1Q9vKlgkv66PgRzFaV74yIamMXe9l71dUvbW3X9bNlEzxbbP6lJY0/bf2/72vpzNGWWqUnN9sPAkKRBTL4RzXiry/j5c4BtKZWu4xuM98va7bXTBfZQyuxdTfltVyeDJ9i+kfJQZl9MmZikw9UAAAXSSURBVJo+g/82fiJlLI7lTT31u651+pZ/3/aVDYZ7kqS9Oo981wfD+jpt2wS6DfiByhwM3cNwNzXdXvRX5z/5gcDJtq+WGp3L8yhgIfBcST+lzNp1RIPxVtYuqf9BmZZ1DWXUzb6YSkl/oN/GbmjChpEM07f8Sw33LZ8PLJK0dV2+m0eniJzs7qw/G9HHCadjYJZL+jblgaz3S9qShiZ+r2NAzbb9OpXZ4zbyo5O/N8L2G+rbD0q6kDJJzHn92v+UuZFbBylaCLyS8sDUrcARU6V9c6L6lkvaivJ3MpAZySLGUhPxLsAttu9WGRhwhhsafFDSxbb/sIl9T4SpVNP/KeXmzoXANpTxtefR7OTog3QbA+5b3j2SYefq2VNgJMNae3pMbcd2U+PNR3/tVV9f3GyrziOWSvo7yqRF3c2Bjcw53LSplPTPojRBXEEf278eRwbat3ykkQz7GWMC/V3X+80pwwIP5L5M9MV7u95vThluZTnNTRLTadY8qqusiUlwBmIqNe8MdKS/QRt03/JBj2Q40SR9z/arJ/o4Yvwk7Qh83PabJvpYJoOpVNMf9Eh/A+Uy9+hmwHMptYybbP+uwZCdZqTOSIaraXYkw4HR2hOjd4bs+IMJOpzYcCspkws1QmU+3v9JGUkXyix2n3fD81k0ZSol/b2AP5d0K4MZ6W+gJB1ImRv0x5Rz20nS221/q6GQ/znMSIZfaCjWoHVPjP4g5X5JXyfgiOZI+lcevSfTual7dYMhT6SMOPvZuvzmWva2BmM2Ziol/b4/GPU48yngtbZXwCNjpJ8DNJX0b6RMN/l1Sc+nPPX4Hw3FGrT3AefZ/rWkD1DObdI/bdwi3ZPMPwR81fYPGoz3cq89Ac13JTX5JdOoKZP0p0rXzFHc1Un4VWcC6qZ8wPYZkvai3DT+JKV2s/vom00K/2j79Cl6blPeBIyN9LCkZ9n+MTzSPbzJ8fQbNWWSfgtcJ+lc4HTKpe1hwOWdYWYbGFa280d9EPA522dJ+mCfY0yUqXxuU56kPSmjznbmb+405TbVm+a9wIV1khgo0xge2VCsxk2Z3jtTnaSTR/nY7tP8mV3xzqY8+/A6ymii91Mmn25kntVBmsrn1gaSbgTezWMnYv9VQ/E2B94DdEa5XAqc0HlQcrJJ0o9hSdoC2B+41vbNkrYHXtTgSIYDM5XPrQ0kXeq156xtOt7plIc9T61FbwKm2z5sUMfQT0n6k0StbcynPiHbKe93DT/i8aoOOAjwRmBjyjhU3bPINTK5vaSr170KHK5sskib/uRxCqVHzX6UoSWOAG6Y0COKGKxPrrM8u+u9ae6J3Csl7WH7EgBJu1MmTpqUUtOfJCRdafulXU/Ibgqcn/Fiom0kPdP2LWOV9SFOZxa5TSnj2d9el58BXD9ZRwBITX/y6Dz9d7ekFwI/p/QiiGibMynPVnQ7g3JTvp9eP/Yqk0+S/uSxUNJ04B+BJcCTgQ9M7CFFDE6dNvAFwNadrsrVVjQwgfhUffYnSX/yOIUyGuRMoPNwynYTdjQRg7czpfY9DfijrvJ7gb+YkCOahNKmP0lIOg+4h8f2TV735lbElCbpFbZ/ONHHMVkl6U8SU33o6IheSRqi1Oxn0tVake7LvUnzzuQxpYeOjhiHs4D/Ar7DJB4DZ6Kkpv8419VtbBNgFmWgtSk3dHREryRdZXuXiT6OySo1/ce/KdltLGIDnC3pQNvnTvSBTEap6UfEpCLpXsr8zb+jPL/SuerdakIPbJJITT8iJputKcOQ7GT7WElPB7af4GOaNFLTj4hJRdKJwO+BvW0/rz60+G3bL5/gQ5sUUtOPiMlmd9u7SroSwPYaSZtN9EFNFhtN9AFERIzTg5I2pk6OXvvt/35iD2nySNKPiMnmM8A3gW0lHQd8H/joxB7S5JE2/YiYdOrga/tQeu5cYDtzS/QoST8iokXSvBMR0SJJ+hERLZKkHxHRIkn6EREtkqQfEdEi/x88U72xDMlMQQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.sentiment.value_counts().plot.bar()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Polarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/stb/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py:3940: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  errors=errors)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NaiveBayes Count Score:  0.5575349559813568 \n",
      "\n",
      "[[2546  778  428]\n",
      " [ 998 2354  417]\n",
      " [ 854  797  483]]\n"
     ]
    }
   ],
   "source": [
    "# Create features and target\n",
    "X = df.drop(['sentiment', 'polarity'], axis=1)\n",
    "y = df.polarity\n",
    "\n",
    "# Split training and testing data\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.25, random_state=57)\n",
    "\n",
    "# Initialize count vectorizer\n",
    "count_vectorizer = CountVectorizer(stop_words='english', strip_accents='ascii',\n",
    "                                  min_df=.001)\n",
    "\n",
    "# Create count train and test variables\n",
    "X_train_bow = count_vectorizer.fit_transform(X_train.content)\n",
    "X_test_bow = count_vectorizer.transform(X_test.content)\n",
    "\n",
    "# Convert matrices into a DataFrame\n",
    "X_train_bow_df = pd.DataFrame(X_train_bow.toarray())\n",
    "X_test_bow_df = pd.DataFrame(X_test_bow.toarray())\n",
    "\n",
    "# Map the column names to vocabulary \n",
    "X_train_bow_df.columns = count_vectorizer.get_feature_names()\n",
    "X_test_bow_df.columns = count_vectorizer.get_feature_names()\n",
    "\n",
    "# Drop 'content'\n",
    "X_train.drop('content', axis=1, inplace=True)\n",
    "X_test.drop('content', axis=1, inplace=True)\n",
    "\n",
    "# Merge data\n",
    "X_train = X_train.reset_index(drop=True).join(X_train_bow_df)\n",
    "X_test = X_test.reset_index(drop=True).join(X_test_bow_df)\n",
    "X_test.fillna(0, inplace=True)\n",
    "\n",
    "# Create a MulitnomialNB model\n",
    "clf_nb = MultinomialNB()\n",
    "\n",
    "clf_nb.fit(X_train, y_train)\n",
    "\n",
    "# Run predict on count test data to get predictions\n",
    "count_nb_pred = clf_nb.predict(X_test)\n",
    "\n",
    "# Calculate the accuracy of predictions\n",
    "count_nb_score = clf_nb.score(X_test, y_test)\n",
    "\n",
    "print('NaiveBayes Count Score: ', count_nb_score, '\\n')\n",
    "print(confusion_matrix(y_test, count_nb_pred, labels=y.unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1a19247710>"
      ]
     },
     "execution_count": 248,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEaCAYAAAAfVJzKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAXAElEQVR4nO3df5BdZ33f8fcHgUzCr5p400wkC8kgfogfschiw6QhDdggSmO5iQlyQ8Z06GhIrULqtEU0jJmRQwtm6gydCrAyUUpoHMWBJtk0Ii4BQ4dQg9Y/sCODylp28EbMIJCxKRgb2d/+cY+Z65sr7Vnt7r3as+/XzI7uec7z3P3urOezj5/z3HNSVUiSuusJ4y5AkrS0DHpJ6jiDXpI6zqCXpI4z6CWp4wx6Seq4J467gEFnnXVWrV+/ftxlSNKycvPNN3+zqiaGnTvtgn79+vVMT0+PuwxJWlaS/O2JzrVaukmyJcmhJDNJdp6k3yVJKslkX9s7m3GHkrx2fqVLkhZqzhl9klXAbuBCYBY4kGSqqu4c6Pc04G3AF/raNgHbgBcCPwn8VZLnVtUji/cjSJJOps2M/jxgpqoOV9XDwD5g65B+VwFXA9/va9sK7Kuqh6rqbmCmeT9J0oi0Cfo1wL19x7NN2w8l2QycXVX/c75jJUlLq03QZ0jbD++EluQJwG8DvzHfsX3vsT3JdJLpo0ePtihJktRWm6CfBc7uO14LHOk7fhrwIuAzSe4BXg5MNRdk5xoLQFXtqarJqpqcmBi6O0iSdIraBP0BYGOSDUlW07u4OvXYyaq6v6rOqqr1VbUeuAm4qKqmm37bkpyRZAOwEfjiov8UkqQTmnPXTVUdT7IDuAFYBeytqoNJdgHTVTV1krEHk1wP3AkcBy53x40kjVZOtwePTE5O1ig/MLV+51+M7HuNwz3vff24S5A0AklurqrJYedOu0/GSvPhH2ppbt7UTJI6zqCXpI4z6CWp4wx6Seo4g16SOs6gl6SOM+glqeMMeknqOINekjrOoJekjjPoJanjDHpJ6jiDXpI6zqCXpI4z6CWp4wx6Seo4g16SOq5V0CfZkuRQkpkkO4ecf2uSO5LcluRzSTY17euTPNi035bkw4v9A0iSTm7ORwkmWQXsBi4EZoEDSaaq6s6+btdV1Yeb/hcB1wBbmnN3VdW5i1u2JKmtNjP684CZqjpcVQ8D+4Ct/R2q6oG+w6cAp9cTxyVpBWsT9GuAe/uOZ5u2x0lyeZK7gKuBt/Wd2pDk1iSfTfKzw75Bku1JppNMHz16dB7lS5Lm0iboM6Tt783Yq2p3VT0beAfwrqb568C6qtoMXAFcl+TpQ8buqarJqpqcmJhoX70kaU5tgn4WOLvveC1w5CT99wEXA1TVQ1X1reb1zcBdwHNPrVRJ0qloE/QHgI1JNiRZDWwDpvo7JNnYd/h64KtN+0RzMZck5wAbgcOLUbgkqZ05d91U1fEkO4AbgFXA3qo6mGQXMF1VU8COJBcAPwDuAy5rhr8S2JXkOPAI8NaqOrYUP4gkabg5gx6gqvYD+wfarux7/fYTjPs48PGFFChJWhg/GStJHWfQS1LHGfSS1HEGvSR1nEEvSR1n0EtSxxn0ktRxBr0kdZxBL0kdZ9BLUscZ9JLUcQa9JHWcQS9JHWfQS1LHGfSS1HEGvSR1nEEvSR3XKuiTbElyKMlMkp1Dzr81yR1JbkvyuSSb+s69sxl3KMlrF7N4SdLc5gz65uHeu4HXAZuAS/uDvHFdVb24qs4FrgauacZuovcw8RcCW4APPvawcEnSaLSZ0Z8HzFTV4ap6GNgHbO3vUFUP9B0+Bajm9VZgX1U9VFV3AzPN+0mSRqTNw8HXAPf2Hc8C5w92SnI5cAWwGnhV39ibBsauOaVKJUmnpM2MPkPa6u81VO2uqmcD7wDeNZ+xSbYnmU4yffTo0RYlSZLaahP0s8DZfcdrgSMn6b8PuHg+Y6tqT1VNVtXkxMREi5IkSW21CfoDwMYkG5Kspndxdaq/Q5KNfYevB77avJ4CtiU5I8kGYCPwxYWXLUlqa841+qo6nmQHcAOwCthbVQeT7AKmq2oK2JHkAuAHwH3AZc3Yg0muB+4EjgOXV9UjS/SzSJKGaHMxlqraD+wfaLuy7/XbTzL2PcB7TrVASdLC+MlYSeo4g16SOs6gl6SOM+glqeMMeknqOINekjrOoJekjjPoJanjDHpJ6jiDXpI6zqCXpI4z6CWp4wx6Seo4g16SOs6gl6SOM+glqeMMeknqOINekjquVdAn2ZLkUJKZJDuHnL8iyZ1Jbk/yqSTP6jv3SJLbmq+pwbGSpKU15zNjk6wCdgMXArPAgSRTVXVnX7dbgcmq+l6SXwOuBt7YnHuwqs5d5LolSS21mdGfB8xU1eGqehjYB2zt71BVN1bV95rDm4C1i1umJOlUtQn6NcC9fcezTduJvAX4RN/xk5NMJ7kpycXDBiTZ3vSZPnr0aIuSJEltzbl0A2RIWw3tmLwJmAR+rq95XVUdSXIO8Okkd1TVXY97s6o9wB6AycnJoe8tSTo1bWb0s8DZfcdrgSODnZJcAPwmcFFVPfRYe1Udaf49DHwG2LyAeiVJ89Qm6A8AG5NsSLIa2AY8bvdMks3AtfRC/ht97WcmOaN5fRbwM0D/RVxJ0hKbc+mmqo4n2QHcAKwC9lbVwSS7gOmqmgLeDzwV+OMkAF+rqouAFwDXJnmU3h+V9w7s1pEkLbE2a/RU1X5g/0DblX2vLzjBuM8DL15IgZKkhfGTsZLUcQa9JHWcQS9JHWfQS1LHGfSS1HEGvSR1nEEvSR1n0EtSxxn0ktRxBr0kdZxBL0kd1+peN5K0FNbv/Itxl7Bk7nnv68ddwg85o5ekjjPoJanjDHpJ6jiDXpI6zqCXpI5rFfRJtiQ5lGQmyc4h569IcmeS25N8Ksmz+s5dluSrzddli1m8JGlucwZ9klXAbuB1wCbg0iSbBrrdCkxW1UuAjwFXN2OfCbwbOB84D3h3kjMXr3xJ0lzazOjPA2aq6nBVPQzsA7b2d6iqG6vqe83hTcDa5vVrgU9W1bGqug/4JLBlcUqXJLXRJujXAPf2Hc82bSfyFuAT8xmbZHuS6STTR48ebVGSJKmtNkGfIW01tGPyJmASeP98xlbVnqqarKrJiYmJFiVJktpqE/SzwNl9x2uBI4OdklwA/CZwUVU9NJ+xkqSl0yboDwAbk2xIshrYBkz1d0iyGbiWXsh/o+/UDcBrkpzZXIR9TdMmSRqROW9qVlXHk+ygF9CrgL1VdTDJLmC6qqboLdU8FfjjJABfq6qLqupYkqvo/bEA2FVVx5bkJ5EkDdXq7pVVtR/YP9B2Zd/rC04ydi+w91QLlCQtjJ+MlaSOM+glqeMMeknqOINekjrOoJekjjPoJanjDHpJ6jiDXpI6zqCXpI4z6CWp4wx6Seo4g16SOs6gl6SOM+glqeMMeknqOINekjrOoJekjmsV9Em2JDmUZCbJziHnX5nkliTHk1wycO6RJLc1X1ODYyVJS2vORwkmWQXsBi4EZoEDSaaq6s6+bl8D3gz82yFv8WBVnbsItUqSTkGbZ8aeB8xU1WGAJPuArcAPg76q7mnOPboENUqSFqDN0s0a4N6+49mmra0nJ5lOclOSi+dVnSRpwdrM6DOkrebxPdZV1ZEk5wCfTnJHVd31uG+QbAe2A6xbt24eby1JmkubGf0scHbf8VrgSNtvUFVHmn8PA58BNg/ps6eqJqtqcmJiou1bS5JaaBP0B4CNSTYkWQ1sA1rtnklyZpIzmtdnAT9D39q+JGnpzRn0VXUc2AHcAHwZuL6qDibZleQigCQvSzILvAG4NsnBZvgLgOkkXwJuBN47sFtHkrTE2qzRU1X7gf0DbVf2vT5Ab0lncNzngRcvsEZJ0gL4yVhJ6jiDXpI6zqCXpI4z6CWp4wx6Seo4g16SOs6gl6SOM+glqeMMeknqOINekjrOoJekjjPoJanjDHpJ6jiDXpI6zqCXpI4z6CWp4wx6Seo4g16SOq5V0CfZkuRQkpkkO4ecf2WSW5IcT3LJwLnLkny1+bpssQqXJLUzZ9AnWQXsBl4HbAIuTbJpoNvXgDcD1w2MfSbwbuB84Dzg3UnOXHjZkqS22szozwNmqupwVT0M7AO29neoqnuq6nbg0YGxrwU+WVXHquo+4JPAlkWoW5LUUpugXwPc23c827S10Wpsku1JppNMHz16tOVbS5LaaBP0GdJWLd+/1diq2lNVk1U1OTEx0fKtJUlttAn6WeDsvuO1wJGW77+QsZKkRdAm6A8AG5NsSLIa2AZMtXz/G4DXJDmzuQj7mqZNkjQicwZ9VR0HdtAL6C8D11fVwSS7klwEkORlSWaBNwDXJjnYjD0GXEXvj8UBYFfTJkkakSe26VRV+4H9A21X9r0+QG9ZZtjYvcDeBdQoSVoAPxkrSR1n0EtSxxn0ktRxBr0kdZxBL0kdZ9BLUscZ9JLUcQa9JHWcQS9JHWfQS1LHGfSS1HEGvSR1nEEvSR1n0EtSxxn0ktRxBr0kdZxBL0kd1yrok2xJcijJTJKdQ86fkeSPmvNfSLK+aV+f5MEktzVfH17c8iVJc5nzUYJJVgG7gQuBWeBAkqmqurOv21uA+6rqOUm2Ae8D3ticu6uqzl3kuiVJLbWZ0Z8HzFTV4ap6GNgHbB3osxX4SPP6Y8Crk2TxypQknao2Qb8GuLfveLZpG9qnqo4D9wM/1pzbkOTWJJ9N8rMLrFeSNE9zLt0Aw2bm1bLP14F1VfWtJD8N/GmSF1bVA48bnGwHtgOsW7euRUmSpLbazOhngbP7jtcCR07UJ8kTgWcAx6rqoar6FkBV3QzcBTx38BtU1Z6qmqyqyYmJifn/FJKkE2oT9AeAjUk2JFkNbAOmBvpMAZc1ry8BPl1VlWSiuZhLknOAjcDhxSldktTGnEs3VXU8yQ7gBmAVsLeqDibZBUxX1RTwu8BHk8wAx+j9MQB4JbAryXHgEeCtVXVsKX4QSdJwbdboqar9wP6Btiv7Xn8feMOQcR8HPr7AGiVJC+AnYyWp4wx6Seo4g16SOs6gl6SOM+glqeMMeknqOINekjrOoJekjjPoJanjDHpJ6jiDXpI6zqCXpI4z6CWp4wx6Seo4g16SOs6gl6SOM+glqeMMeknquFZBn2RLkkNJZpLsHHL+jCR/1Jz/QpL1fefe2bQfSvLaxStdktTGnEGfZBWwG3gdsAm4NMmmgW5vAe6rqucAvw28rxm7id6Dwl8IbAE+2LyfJGlE2szozwNmqupwVT0M7AO2DvTZCnykef0x4NVJ0rTvq6qHqupuYKZ5P0nSiDyxRZ81wL19x7PA+SfqU1XHk9wP/FjTftPA2DWD3yDJdmB7c/j/khxqVf3ydBbwzVF9s7xvVN9pxfD3t3x1/Xf3rBOdaBP0GdJWLfu0GUtV7QH2tKhl2UsyXVWT465Dp8bf3/K1kn93bZZuZoGz+47XAkdO1CfJE4FnAMdajpUkLaE2QX8A2JhkQ5LV9C6uTg30mQIua15fAny6qqpp39bsytkAbAS+uDilS5LamHPppllz3wHcAKwC9lbVwSS7gOmqmgJ+F/hokhl6M/ltzdiDSa4H7gSOA5dX1SNL9LMsFytiiarD/P0tXyv2d5fexFuS1FV+MlaSOs6gl6SOM+glqeMMeqmFJD+S5HnjrkM6FQb9CKTnTUmubI7XJfFWEMtEkl8AbgP+sjk+N8ngFmPptOWumxFI8iHgUeBVVfWCJGcC/6uqXjbm0tRCkpuBVwGfqarNTdvtVfWS8VamE0nyHYZ8Cp/ep/Wrqp4+4pLGqs0tELRw51fVS5PcClBV9zUfPtPycLyq7u/dp0/LQVU9bdw1nE4M+tH4QXN75gJIMkFvhq/l4W+S/HNgVZKNwNuAz4+5Js1Dkh8HnvzYcVV9bYzljJxr9KPxX4A/AX48yXuAzwH/cbwlaR7+Nb1nKjwEXAfcD/z6WCtSK0kuSvJV4G7gs8A9wCfGWtQYuEY/IkmeD7ya3hrhp6rqy2MuSS0l2VxVt467Ds1fki/Ru77yV1W1OcnPA5dW1fY5hnaKM/oRSPIB4JlVtbuq/qshv+xck+QrSa5K8sJxF6N5+UFVfQt4QpInVNWNwLnjLmrUDPrRuAV4V/Ps3PcnWZH3xF6uqurngX8MHAX2JLkjybvGW5Va+naSpwL/G/iDZtJ1fMw1jZxLNyOU5JnAL9G7u+e6qto45pI0T0leDPx74I1V5c6p01ySpwAP0pvU/gq9Z2X8QTPLXzHcdTNazwGeD6ynd+tmLQNJXgC8kd6zFr5F77nJvzHWojSnZqfbn1XVBfR2uX1kjiGdZdCPQJL3Ab8I3AVcD1xVVd8eb1Wah98D/hB4TVX5hLRloqoeSfK9JM+oqvvHXc84GfSjcTfwiqoa2YOJtXiq6uXjrkGn7PvAHUk+CXz3scaqetv4Sho91+iXUJLnV9VXkrx02PmqumXUNam9JNdX1S8nuYPHf5z+sY/RewuE01ySy4Y0V1X9/siLGSNn9EvrCmA78J+HnCt6+3t1+np78+8/HWsVWoh/UFUf6G9I8vYTde4qZ/QjkOTJVfX9udp0ekryvqp6x1xtOv0kuaWqXjrQdutjN6dbKdxHPxrD7ovivVKWjwuHtL1u5FWotSSXJvlzYEOSqb6vG+ntnFpRXLpZQkl+AlgD/EiSzfTWdgGeDvzo2ApTK0l+DfhXwDlJbu879TTgr8dTlVr6PPB14Cwev3T6HeD2oSM6zKWbJdRcCHozMAlM9536DvDfqup/jKMutZPkGcCZwH8Cdvad+k5VHRtPVdL8GfQjkOSXqurj465DC7PSb3W7HA08gGQ18CTguz54RIsmyZuq6r8D65NcMXi+qq4ZQ1map+ZRgtcAPwl8A3gW8GV6ty7WaWzwASRJLgZW3GM8vRi7tJ7S/PtUeuu6g19aHn4LeDnwf6tqA73bTbtGvwxV1Z+yArc1u3QjzSHJdFVNNvc231xVjyb5YlWtuJnhcpPkF/sOn0DvetnPVdUrxlTSWLh0MwJJrqY3K3wQ+Evgp4Bfb5Z1dPobvNXtN1iBt7pdpn6h7/Vxek+Y2jqeUsbHGf0IJLmtqs5N8s+Ai4F/A9xYVT815tLUQnOr2+/T2x67Ym91q+XLGf1oPKn5958Af1hVx5KcrL9OI1X13b7DFXur2+UoyXOBDwH/sKpelOQlwEVV9VtjLm2kvBg7Gn+e5Cv01gc/lWSC3gxRy0CS7yR5YODr3iR/kuSccdenk/od4J3ADwCq6nZ6D/5ZUZzRj0BV7WzuSf9Ac4/s77IC1wmXsWuAI8B19JZvtgE/ARwC9tJ7zKBOTz9aVV8c+D/oFXd9xaAfgSRPAn4VeGXzH9xngQ+PtSjNx5aqOr/veE+Sm6pqV5L/MLaq1MY3kzyb5kNTSS6hd2uEFcWgH40P0Vun/2Bz/KtN278cW0Waj0eT/DLwseb4kr5z7mY4vV0O7AGen+Tv6D0E6FfGW9LouetmBJJ8aXCHzbA2nZ6adfgPAK+gF+w30ds59XfAT1fV58ZYnk4iyRn0/jCvB54JPEDvwSO7xlnXqDmjH41Hkjy7qu6CHwbHI2OuSS1V1WEevx+7nyF/evsz4NvALfSus6xIBv1o/DvgxiSHm+P1wL8YXzmaD7foLWtrq2rLuIsYN7dXjsZfA9cCjzZf1wL/Z6wVaT7cord8fT7Ji8ddxLg5ox+N36e3NnhVc3wp8FHgDWOrSPPhFr3l6x8Bb05yN/AQK/TB7gb9aDxv4MLrjc0NsrQ8uEVv+fKRjxj0o3JrkpdX1U0ASc7H29wuJ27RW6aq6m/HXcPpwO2VI5Dky8DzgMeeSLSO3oMrHmUF/m/kcuMWPS13zuhHY8Vf9V/m3KKnZc0ZvTSHJH9TVS8adx3SqXJ7pTQ3t+hpWXNGL80hyZ3Ac+hdhF2xW/S0fBn00hySPGtYuzs6tFwY9JLUca7RS1LHGfSS1HEGvSR1nEEvSR1n0EtSx/1/H9junVRzDjEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.polarity.value_counts(normalize=True).plot.bar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #from data.helper_functions import plot_confusion_matrix\n",
    "\n",
    "# # Plot the confusion matrix\n",
    "# plot_confusion_matrix(confusion_matrix(y_test, count_nb_pred).round(), classes=y.unique(), title=\"Count NB Confusion Matrix\", normalize=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### `TfidfVectorizer`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create features and target\n",
    "X = df.drop(['sentiment', 'polarity'], axis=1)\n",
    "y = df.sentiment\n",
    "\n",
    "# Split training and testing data\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.25, random_state=57)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize tfidf vectorizer\n",
    "tfidf_vectorizer = TfidfVectorizer(stop_words='english', min_df=100)\n",
    "\n",
    "# Create tfidf train and test variables\n",
    "X_train_tfidf = tfidf_vectorizer.fit_transform(X_train.content) \n",
    "X_test_tfidf = tfidf_vectorizer.transform(X_test.content)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Multinomial Naive Bayes Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_test_tfidf.toarray()[np.isnan(X_test_tfidf.toarray())] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NaiveBayes Tfidf Score:  0.3232522009321595\n"
     ]
    }
   ],
   "source": [
    "# Create a MulitnomialNB model\n",
    "tfidf_nb = MultinomialNB()\n",
    "\n",
    "tfidf_nb.fit(X_train_tfidf, y_train)\n",
    "\n",
    "# Run predict on TF-IDF test data to get predictions\n",
    "tfidf_nb_pred = tfidf_nb.predict(X_test_tfidf)\n",
    "\n",
    "# Calculate the accuracy of predictions\n",
    "tfidf_nb_score = tfidf_nb.score(X_test_tfidf, y_test)\n",
    "\n",
    "print('NaiveBayes Tfidf Score: ', tfidf_nb_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>anger</th>\n",
       "      <th>boredom</th>\n",
       "      <th>enthusiasm</th>\n",
       "      <th>fun</th>\n",
       "      <th>happiness</th>\n",
       "      <th>hate</th>\n",
       "      <th>love</th>\n",
       "      <th>neutral</th>\n",
       "      <th>relief</th>\n",
       "      <th>sadness</th>\n",
       "      <th>surprise</th>\n",
       "      <th>worry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>anger</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>boredom</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>enthusiasm</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>98</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fun</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>94</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>196</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>happiness</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>350</td>\n",
       "      <td>0</td>\n",
       "      <td>129</td>\n",
       "      <td>491</td>\n",
       "      <td>3</td>\n",
       "      <td>13</td>\n",
       "      <td>4</td>\n",
       "      <td>301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hate</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>18</td>\n",
       "      <td>6</td>\n",
       "      <td>111</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>love</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>170</td>\n",
       "      <td>0</td>\n",
       "      <td>277</td>\n",
       "      <td>281</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "      <td>175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>neutral</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>140</td>\n",
       "      <td>4</td>\n",
       "      <td>51</td>\n",
       "      <td>1282</td>\n",
       "      <td>3</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>602</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>relief</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>51</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>177</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sadness</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>42</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>409</td>\n",
       "      <td>0</td>\n",
       "      <td>158</td>\n",
       "      <td>1</td>\n",
       "      <td>651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>surprise</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>51</td>\n",
       "      <td>0</td>\n",
       "      <td>32</td>\n",
       "      <td>237</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>11</td>\n",
       "      <td>211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>worry</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>5</td>\n",
       "      <td>40</td>\n",
       "      <td>796</td>\n",
       "      <td>2</td>\n",
       "      <td>102</td>\n",
       "      <td>1</td>\n",
       "      <td>1017</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            anger  boredom  enthusiasm  fun  happiness  hate  love  neutral  \\\n",
       "anger           0        0           0    0          1     0     3       10   \n",
       "boredom         0        0           0    0          1     0     2       21   \n",
       "enthusiasm      0        0           0    0         25     0    11       98   \n",
       "fun             0        0           0    0         94     0    10      196   \n",
       "happiness       0        0           0    0        350     0   129      491   \n",
       "hate            0        0           0    0          9    18     6      111   \n",
       "love            0        0           0    0        170     0   277      281   \n",
       "neutral         0        0           0    0        140     4    51     1282   \n",
       "relief          0        0           0    0         51     1    13      177   \n",
       "sadness         0        0           0    0         42     9    24      409   \n",
       "surprise        0        0           0    0         51     0    32      237   \n",
       "worry           0        0           0    0        120     5    40      796   \n",
       "\n",
       "            relief  sadness  surprise  worry  \n",
       "anger            0        1         0     13  \n",
       "boredom          0        4         0     19  \n",
       "enthusiasm       0        5         0     58  \n",
       "fun              0        6         0    125  \n",
       "happiness        3       13         4    301  \n",
       "hate             0       18         0    138  \n",
       "love             1       16         2    175  \n",
       "neutral          3       50         2    602  \n",
       "relief           8       13         0    106  \n",
       "sadness          0      158         1    651  \n",
       "surprise         2       15        11    211  \n",
       "worry            2      102         1   1017  "
      ]
     },
     "execution_count": 255,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print Confusion Matrix\n",
    "labels = np.unique(y_test)\n",
    "cm =  confusion_matrix(y_test, tfidf_nb_pred, labels=labels)\n",
    "pd.DataFrame(cm, index=labels, columns=labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Linear SVC Model (Emotions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LinearSVC Score:   0.343\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>anger</th>\n",
       "      <th>boredom</th>\n",
       "      <th>enthusiasm</th>\n",
       "      <th>fun</th>\n",
       "      <th>happiness</th>\n",
       "      <th>hate</th>\n",
       "      <th>love</th>\n",
       "      <th>neutral</th>\n",
       "      <th>relief</th>\n",
       "      <th>sadness</th>\n",
       "      <th>surprise</th>\n",
       "      <th>worry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>anger</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>boredom</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>enthusiasm</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>36</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>87</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fun</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>139</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>happiness</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>481</td>\n",
       "      <td>0</td>\n",
       "      <td>146</td>\n",
       "      <td>383</td>\n",
       "      <td>8</td>\n",
       "      <td>30</td>\n",
       "      <td>9</td>\n",
       "      <td>230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hate</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>48</td>\n",
       "      <td>8</td>\n",
       "      <td>98</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>love</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>198</td>\n",
       "      <td>1</td>\n",
       "      <td>360</td>\n",
       "      <td>218</td>\n",
       "      <td>2</td>\n",
       "      <td>28</td>\n",
       "      <td>5</td>\n",
       "      <td>106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>neutral</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>216</td>\n",
       "      <td>5</td>\n",
       "      <td>64</td>\n",
       "      <td>1241</td>\n",
       "      <td>4</td>\n",
       "      <td>86</td>\n",
       "      <td>4</td>\n",
       "      <td>510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>relief</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>91</td>\n",
       "      <td>5</td>\n",
       "      <td>17</td>\n",
       "      <td>132</td>\n",
       "      <td>17</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sadness</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>67</td>\n",
       "      <td>26</td>\n",
       "      <td>41</td>\n",
       "      <td>373</td>\n",
       "      <td>0</td>\n",
       "      <td>259</td>\n",
       "      <td>5</td>\n",
       "      <td>521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>surprise</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>75</td>\n",
       "      <td>0</td>\n",
       "      <td>39</td>\n",
       "      <td>211</td>\n",
       "      <td>3</td>\n",
       "      <td>26</td>\n",
       "      <td>22</td>\n",
       "      <td>183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>worry</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>179</td>\n",
       "      <td>21</td>\n",
       "      <td>60</td>\n",
       "      <td>747</td>\n",
       "      <td>9</td>\n",
       "      <td>167</td>\n",
       "      <td>13</td>\n",
       "      <td>886</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            anger  boredom  enthusiasm  fun  happiness  hate  love  neutral  \\\n",
       "anger           0        0           0    0          1     0     3       11   \n",
       "boredom         0        0           0    0          2     0     2       22   \n",
       "enthusiasm      0        0           0    0         36     0    13       87   \n",
       "fun             0        0           0    2        139     1    18      150   \n",
       "happiness       0        0           0    4        481     0   146      383   \n",
       "hate            0        0           0    1         17    48     8       98   \n",
       "love            0        0           0    4        198     1   360      218   \n",
       "neutral         0        0           0    4        216     5    64     1241   \n",
       "relief          0        0           0    0         91     5    17      132   \n",
       "sadness         0        0           0    2         67    26    41      373   \n",
       "surprise        0        0           0    0         75     0    39      211   \n",
       "worry           0        0           0    1        179    21    60      747   \n",
       "\n",
       "            relief  sadness  surprise  worry  \n",
       "anger            1        1         0     11  \n",
       "boredom          0        5         0     16  \n",
       "enthusiasm       0        8         1     52  \n",
       "fun              0       11         3    107  \n",
       "happiness        8       30         9    230  \n",
       "hate             0       23         0    105  \n",
       "love             2       28         5    106  \n",
       "neutral          4       86         4    510  \n",
       "relief          17       11         0     96  \n",
       "sadness          0      259         5    521  \n",
       "surprise         3       26        22    183  \n",
       "worry            9      167        13    886  "
      ]
     },
     "execution_count": 256,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a LinearSVC model\n",
    "tfidf_svc = LinearSVC()\n",
    "\n",
    "# Train model\n",
    "tfidf_svc.fit(X_train_tfidf, y_train)\n",
    "\n",
    "# Run predict on tfidf test data to get predictions\n",
    "tfidf_svc_pred = tfidf_svc.predict(X_test_tfidf)\n",
    "\n",
    "# Calculate accuracy using the metrics module\n",
    "tfidf_svc_score = tfidf_svc.score(X_test_tfidf, y_test)\n",
    "\n",
    "print(\"LinearSVC Score:   %0.3f\" % tfidf_svc_score)\n",
    "\n",
    "# Print Confusion Matrix\n",
    "labels = np.unique(y_test)\n",
    "cm =  confusion_matrix(y_test, tfidf_svc_pred, labels=labels)\n",
    "pd.DataFrame(cm, index=labels, columns=labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Linear SVC Model (Polarity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LinearSVC Score:   0.561\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>negative</th>\n",
       "      <th>neutral</th>\n",
       "      <th>positive</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>negative</th>\n",
       "      <td>2572</td>\n",
       "      <td>298</td>\n",
       "      <td>882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>neutral</th>\n",
       "      <td>999</td>\n",
       "      <td>423</td>\n",
       "      <td>712</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>positive</th>\n",
       "      <td>1099</td>\n",
       "      <td>253</td>\n",
       "      <td>2417</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          negative  neutral  positive\n",
       "negative      2572      298       882\n",
       "neutral        999      423       712\n",
       "positive      1099      253      2417"
      ]
     },
     "execution_count": 261,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create features and target\n",
    "X = df.drop(['sentiment', 'polarity'], axis=1)\n",
    "y = df.polarity\n",
    "\n",
    "# Split training and testing data\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.25, random_state=57)\n",
    "\n",
    "# Initialize tfidf vectorizer\n",
    "tfidf_vectorizer = TfidfVectorizer(stop_words='english', min_df=100)\n",
    "\n",
    "# Create tfidf train and test variables\n",
    "X_train_tfidf = tfidf_vectorizer.fit_transform(X_train.content) \n",
    "X_test_tfidf = tfidf_vectorizer.transform(X_test.content)\n",
    "\n",
    "# Create a LinearSVC model\n",
    "tfidf_svc = LinearSVC()\n",
    "\n",
    "# Train model\n",
    "tfidf_svc.fit(X_train_tfidf, y_train)\n",
    "\n",
    "# Run predict on tfidf test data to get predictions\n",
    "tfidf_svc_pred = tfidf_svc.predict(X_test_tfidf)\n",
    "\n",
    "# Calculate accuracy using the metrics module\n",
    "tfidf_svc_score = tfidf_svc.score(X_test_tfidf, y_test)\n",
    "\n",
    "print(\"LinearSVC Score:   %0.3f\" % tfidf_svc_score)\n",
    "\n",
    "# Print Confusion Matrix\n",
    "labels = np.unique(y_test)\n",
    "cm =  confusion_matrix(y_test, tfidf_svc_pred, labels=labels)\n",
    "pd.DataFrame(cm, index=labels, columns=labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### MultinomialNB with Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.2995339202485759\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>anger</th>\n",
       "      <th>boredom</th>\n",
       "      <th>enthusiasm</th>\n",
       "      <th>fun</th>\n",
       "      <th>happiness</th>\n",
       "      <th>hate</th>\n",
       "      <th>love</th>\n",
       "      <th>neutral</th>\n",
       "      <th>relief</th>\n",
       "      <th>sadness</th>\n",
       "      <th>surprise</th>\n",
       "      <th>worry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>anger</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>boredom</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>enthusiasm</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>94</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fun</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>195</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>happiness</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>157</td>\n",
       "      <td>0</td>\n",
       "      <td>36</td>\n",
       "      <td>549</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hate</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>56</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>love</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>74</td>\n",
       "      <td>0</td>\n",
       "      <td>130</td>\n",
       "      <td>328</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>neutral</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>39</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>1034</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>1031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>relief</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>157</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sadness</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>254</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>1021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>surprise</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>181</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>worry</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>494</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>1556</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            anger  boredom  enthusiasm  fun  happiness  hate  love  neutral  \\\n",
       "anger           0        0           0    0          0     0     0        8   \n",
       "boredom         0        0           0    0          0     0     0       13   \n",
       "enthusiasm      0        0           0    0          2     0     2       94   \n",
       "fun             0        0           0    0         29     0     2      195   \n",
       "happiness       0        0           0    0        157     0    36      549   \n",
       "hate            0        0           0    0          1     0     0       56   \n",
       "love            0        0           0    0         74     0   130      328   \n",
       "neutral         0        0           0    0         39     0    20     1034   \n",
       "relief          0        0           0    0          7     0     5      157   \n",
       "sadness         0        0           0    0          4     0     0      254   \n",
       "surprise        0        0           0    0         17     0    15      181   \n",
       "worry           0        0           0    0         18     0     6      494   \n",
       "\n",
       "            relief  sadness  surprise  worry  \n",
       "anger            0        0         0     20  \n",
       "boredom          0        0         0     34  \n",
       "enthusiasm       0        1         0     98  \n",
       "fun              0        1         0    204  \n",
       "happiness        0        0         0    549  \n",
       "hate             0        4         0    239  \n",
       "love             0        0         0    390  \n",
       "neutral          0       10         0   1031  \n",
       "relief           0        0         0    200  \n",
       "sadness          0       15         0   1021  \n",
       "surprise         0        1         0    345  \n",
       "worry            0        9         0   1556  "
      ]
     },
     "execution_count": 257,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "\n",
    "text_clf = Pipeline([\n",
    "    ('vect', CountVectorizer()),\n",
    "    ('tfidf', TfidfTransformer()),\n",
    "    ('clf', MultinomialNB()),\n",
    "])\n",
    "\n",
    "text_clf.fit(X_train.content, y_train)\n",
    "\n",
    "print(text_clf.score(X_test.content, y_test))\n",
    "\n",
    "y_pred = text_clf.predict(X_test.content)\n",
    "\n",
    "# Print Confusion Matrix\n",
    "labels = np.unique(y_test)\n",
    "cm =  confusion_matrix(y_test, y_pred, labels=labels)\n",
    "pd.DataFrame(cm, index=labels, columns=labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "parameters = {\n",
    "    'vect__stop_words': ['english', None],\n",
    "    'vect__max_df': [0.7, 0.8, 0.9, 1.0],\n",
    "    'vect__min_df': [10, .001, .01, 0.1],    \n",
    "    'clf__alpha': (0.1, 1.0)\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(text_clf, parameters, cv=3,\n",
    "                           n_jobs=-1, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 64 candidates, totalling 192 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   15.0s\n",
      "[Parallel(n_jobs=-1)]: Done 192 out of 192 | elapsed:   56.8s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score 0.3328845157949249\n",
      "-----\n",
      "GridSearchCV:\n",
      "Best score: 0.331\n",
      "Best parameters set:\n",
      "\tclf__alpha: 0.1\n",
      "\tvect__max_df: 0.7\n",
      "\tvect__min_df: 0.001\n",
      "\tvect__stop_words: None\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>anger</th>\n",
       "      <th>boredom</th>\n",
       "      <th>enthusiasm</th>\n",
       "      <th>fun</th>\n",
       "      <th>happiness</th>\n",
       "      <th>hate</th>\n",
       "      <th>love</th>\n",
       "      <th>neutral</th>\n",
       "      <th>relief</th>\n",
       "      <th>sadness</th>\n",
       "      <th>surprise</th>\n",
       "      <th>worry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>anger</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>boredom</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>enthusiasm</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>103</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fun</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>91</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>203</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>happiness</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>385</td>\n",
       "      <td>0</td>\n",
       "      <td>118</td>\n",
       "      <td>488</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>284</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hate</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>14</td>\n",
       "      <td>4</td>\n",
       "      <td>86</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "      <td>163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>love</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>162</td>\n",
       "      <td>0</td>\n",
       "      <td>275</td>\n",
       "      <td>277</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>neutral</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>163</td>\n",
       "      <td>0</td>\n",
       "      <td>54</td>\n",
       "      <td>1217</td>\n",
       "      <td>1</td>\n",
       "      <td>55</td>\n",
       "      <td>2</td>\n",
       "      <td>642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>relief</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>51</td>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "      <td>166</td>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sadness</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>7</td>\n",
       "      <td>16</td>\n",
       "      <td>366</td>\n",
       "      <td>0</td>\n",
       "      <td>155</td>\n",
       "      <td>0</td>\n",
       "      <td>716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>surprise</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>53</td>\n",
       "      <td>1</td>\n",
       "      <td>31</td>\n",
       "      <td>215</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>4</td>\n",
       "      <td>239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>worry</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>90</td>\n",
       "      <td>3</td>\n",
       "      <td>33</td>\n",
       "      <td>690</td>\n",
       "      <td>1</td>\n",
       "      <td>106</td>\n",
       "      <td>0</td>\n",
       "      <td>1160</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            anger  boredom  enthusiasm  fun  happiness  hate  love  neutral  \\\n",
       "anger           0        0           0    0          1     0     2        8   \n",
       "boredom         0        0           0    0          0     1     1       15   \n",
       "enthusiasm      0        0           0    0         25     0     6      103   \n",
       "fun             0        0           0    1         91     0     8      203   \n",
       "happiness       0        0           0    0        385     0   118      488   \n",
       "hate            0        0           0    0          5    14     4       86   \n",
       "love            0        0           0    0        162     0   275      277   \n",
       "neutral         0        0           0    0        163     0    54     1217   \n",
       "relief          0        0           0    0         51     0    14      166   \n",
       "sadness         0        0           0    0         34     7    16      366   \n",
       "surprise        0        0           0    0         53     1    31      215   \n",
       "worry           0        0           0    0         90     3    33      690   \n",
       "\n",
       "            relief  sadness  surprise  worry  \n",
       "anger            0        0         0     17  \n",
       "boredom          0        4         0     26  \n",
       "enthusiasm       0        4         0     59  \n",
       "fun              0        5         0    123  \n",
       "happiness        0       15         1    284  \n",
       "hate             0       28         0    163  \n",
       "love             0       16         1    191  \n",
       "neutral          1       55         2    642  \n",
       "relief           3       12         0    123  \n",
       "sadness          0      155         0    716  \n",
       "surprise         0       16         4    239  \n",
       "worry            1      106         0   1160  "
      ]
     },
     "execution_count": 259,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.fit(X_train.content, y_train)\n",
    "\n",
    "print('score', grid_search.score(X_test.content, y_test))\n",
    "print('-----')\n",
    "\n",
    "print('GridSearchCV:')\n",
    "y_pred = grid_search.predict(X_test.content)\n",
    "\n",
    "print(\"Best score: %0.3f\" % grid_search.best_score_)\n",
    "print(\"Best parameters set:\")\n",
    "best_parameters = grid_search.best_estimator_.get_params()\n",
    "for param_name in sorted(parameters.keys()):\n",
    "    print(\"\\t%s: %r\" % (param_name, best_parameters[param_name]))\n",
    "\n",
    "# Print Confusion Matrix\n",
    "labels = np.unique(y_test)\n",
    "cm =  confusion_matrix(y_test, y_pred, labels=labels)\n",
    "pd.DataFrame(cm, index=labels, columns=labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### LinearSVC Classifier with Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 36 candidates, totalling 108 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:  1.6min\n",
      "[Parallel(n_jobs=-1)]: Done 108 out of 108 | elapsed:  2.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score 0.3554634904194718\n",
      "-----\n",
      "GridSearchCV:\n",
      "Best score: 0.347\n",
      "Best parameters set:\n",
      "\tclf__class_weight: None\n",
      "\tvect__max_df: 0.6\n",
      "\tvect__min_df: 0.001\n",
      "\tvect__stop_words: None\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>anger</th>\n",
       "      <th>boredom</th>\n",
       "      <th>enthusiasm</th>\n",
       "      <th>fun</th>\n",
       "      <th>happiness</th>\n",
       "      <th>hate</th>\n",
       "      <th>love</th>\n",
       "      <th>neutral</th>\n",
       "      <th>relief</th>\n",
       "      <th>sadness</th>\n",
       "      <th>surprise</th>\n",
       "      <th>worry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>anger</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>boredom</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>enthusiasm</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>34</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>91</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fun</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>143</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>136</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>3</td>\n",
       "      <td>104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>happiness</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>493</td>\n",
       "      <td>1</td>\n",
       "      <td>148</td>\n",
       "      <td>354</td>\n",
       "      <td>14</td>\n",
       "      <td>53</td>\n",
       "      <td>14</td>\n",
       "      <td>204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hate</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>61</td>\n",
       "      <td>6</td>\n",
       "      <td>66</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>1</td>\n",
       "      <td>119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>love</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>195</td>\n",
       "      <td>1</td>\n",
       "      <td>373</td>\n",
       "      <td>192</td>\n",
       "      <td>4</td>\n",
       "      <td>32</td>\n",
       "      <td>6</td>\n",
       "      <td>111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>neutral</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>230</td>\n",
       "      <td>9</td>\n",
       "      <td>77</td>\n",
       "      <td>1184</td>\n",
       "      <td>8</td>\n",
       "      <td>125</td>\n",
       "      <td>16</td>\n",
       "      <td>471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>relief</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>86</td>\n",
       "      <td>4</td>\n",
       "      <td>24</td>\n",
       "      <td>120</td>\n",
       "      <td>17</td>\n",
       "      <td>23</td>\n",
       "      <td>2</td>\n",
       "      <td>92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sadness</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>68</td>\n",
       "      <td>30</td>\n",
       "      <td>39</td>\n",
       "      <td>305</td>\n",
       "      <td>1</td>\n",
       "      <td>320</td>\n",
       "      <td>9</td>\n",
       "      <td>519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>surprise</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>82</td>\n",
       "      <td>5</td>\n",
       "      <td>49</td>\n",
       "      <td>181</td>\n",
       "      <td>3</td>\n",
       "      <td>39</td>\n",
       "      <td>22</td>\n",
       "      <td>175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>worry</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>163</td>\n",
       "      <td>29</td>\n",
       "      <td>57</td>\n",
       "      <td>605</td>\n",
       "      <td>10</td>\n",
       "      <td>242</td>\n",
       "      <td>15</td>\n",
       "      <td>956</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            anger  boredom  enthusiasm  fun  happiness  hate  love  neutral  \\\n",
       "anger           0        0           0    0          2     0     2        5   \n",
       "boredom         0        0           0    0          3     3     3       14   \n",
       "enthusiasm      0        0           0    1         34     0    12       91   \n",
       "fun             0        0           0    6        143     2    17      136   \n",
       "happiness       0        0           0   10        493     1   148      354   \n",
       "hate            0        0           0    0         13    61     6       66   \n",
       "love            0        0           0    8        195     1   373      192   \n",
       "neutral         0        1           1   12        230     9    77     1184   \n",
       "relief          0        0           0    1         86     4    24      120   \n",
       "sadness         0        0           0    3         68    30    39      305   \n",
       "surprise        0        0           0    3         82     5    49      181   \n",
       "worry           0        1           1    4        163    29    57      605   \n",
       "\n",
       "            relief  sadness  surprise  worry  \n",
       "anger            1        1         0     17  \n",
       "boredom          0        7         0     17  \n",
       "enthusiasm       1        9         1     48  \n",
       "fun              1       19         3    104  \n",
       "happiness       14       53        14    204  \n",
       "hate             0       34         1    119  \n",
       "love             4       32         6    111  \n",
       "neutral          8      125        16    471  \n",
       "relief          17       23         2     92  \n",
       "sadness          1      320         9    519  \n",
       "surprise         3       39        22    175  \n",
       "worry           10      242        15    956  "
      ]
     },
     "execution_count": 260,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svc_pipeline = Pipeline([\n",
    "    ('vect', CountVectorizer()),\n",
    "    ('tfidf', TfidfTransformer()),\n",
    "    ('clf', LinearSVC()),\n",
    "])\n",
    "\n",
    "parameters = {\n",
    "    'vect__stop_words': ['english', None],\n",
    "    'vect__max_df': [0.6, 0.75, 1.0],\n",
    "    'vect__min_df': [0.001, 0.1, 0.01],\n",
    "    'clf__class_weight': ['balanced', None]\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(svc_pipeline, parameters, cv=3,\n",
    "                           n_jobs=-1, verbose=1)\n",
    "\n",
    "grid_search.fit(X_train.content, y_train)\n",
    "\n",
    "print('score', grid_search.score(X_test.content, y_test))\n",
    "print('-----')\n",
    "\n",
    "print('GridSearchCV:')\n",
    "y_pred = grid_search.predict(X_test.content)\n",
    "\n",
    "print(\"Best score: %0.3f\" % grid_search.best_score_)\n",
    "print(\"Best parameters set:\")\n",
    "best_parameters = grid_search.best_estimator_.get_params()\n",
    "for param_name in sorted(parameters.keys()):\n",
    "    print(\"\\t%s: %r\" % (param_name, best_parameters[param_name]))\n",
    "\n",
    "# Print Confusion Matrix\n",
    "labels = np.unique(y_test)\n",
    "cm =  confusion_matrix(y_test, y_pred, labels=labels)\n",
    "pd.DataFrame(cm, index=labels, columns=labels)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}