spambase.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9adcf47c",
   "metadata": {},
   "source": [
    "# Rebecca Weiss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f40b9dc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.metrics import mean_squared_error, classification_report, accuracy_score, roc_auc_score, f1_score\n",
    "from sklearn.model_selection import train_test_split, cross_val_score, cross_validate\n",
    "from sklearn.ensemble import BaggingClassifier, RandomForestClassifier\n",
    "from sklearn.tree import DecisionTreeRegressor, DecisionTreeClassifier\n",
    "from sklearn.utils import resample\n",
    "import random\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99657dd7",
   "metadata": {},
   "source": [
    "# I will implement the random forest algorithm from scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "e7d5bfd7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word_freq_make:</th>\n",
       "      <th>word_freq_address:</th>\n",
       "      <th>word_freq_all:</th>\n",
       "      <th>word_freq_3d:</th>\n",
       "      <th>word_freq_our:</th>\n",
       "      <th>word_freq_over:</th>\n",
       "      <th>word_freq_remove:</th>\n",
       "      <th>word_freq_internet:</th>\n",
       "      <th>word_freq_order:</th>\n",
       "      <th>word_freq_mail:</th>\n",
       "      <th>...</th>\n",
       "      <th>char_freq_;:</th>\n",
       "      <th>char_freq_(:</th>\n",
       "      <th>char_freq_[:</th>\n",
       "      <th>char_freq_!:</th>\n",
       "      <th>char_freq_$:</th>\n",
       "      <th>char_freq_#:</th>\n",
       "      <th>capital_run_length_average:</th>\n",
       "      <th>capital_run_length_longest:</th>\n",
       "      <th>capital_run_length_total:</th>\n",
       "      <th>spam</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1698</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.87</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.87</td>\n",
       "      <td>0.00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.042</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.085</td>\n",
       "      <td>0.000</td>\n",
       "      <td>5.145</td>\n",
       "      <td>33</td>\n",
       "      <td>247</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1437</th>\n",
       "      <td>0.76</td>\n",
       "      <td>0.19</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.19</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.31</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.087</td>\n",
       "      <td>0.054</td>\n",
       "      <td>0.439</td>\n",
       "      <td>0.241</td>\n",
       "      <td>0.065</td>\n",
       "      <td>3.702</td>\n",
       "      <td>45</td>\n",
       "      <td>1070</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1808</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.077</td>\n",
       "      <td>0.038</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.038</td>\n",
       "      <td>2.600</td>\n",
       "      <td>42</td>\n",
       "      <td>182</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2353</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.75</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.955</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.500</td>\n",
       "      <td>5</td>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>523</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.18</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.11</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.057</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.420</td>\n",
       "      <td>0.191</td>\n",
       "      <td>0.210</td>\n",
       "      <td>8.026</td>\n",
       "      <td>283</td>\n",
       "      <td>1509</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 58 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      word_freq_make:  word_freq_address:  word_freq_all:  word_freq_3d:  \\\n",
       "1698             0.00                0.00            0.29            0.0   \n",
       "1437             0.76                0.19            0.38            0.0   \n",
       "1808             0.00                0.00            0.00            0.0   \n",
       "2353             0.00                0.00            0.00            0.0   \n",
       "523              0.00                0.11            0.35            0.0   \n",
       "\n",
       "      word_freq_our:  word_freq_over:  word_freq_remove:  word_freq_internet:  \\\n",
       "1698            0.87             0.00               0.29                 0.00   \n",
       "1437            0.19             0.12               0.00                 0.25   \n",
       "1808            0.00             0.23               0.00                 0.00   \n",
       "2353            1.75             0.00               0.00                 0.00   \n",
       "523             1.18             0.47               0.23                 0.35   \n",
       "\n",
       "      word_freq_order:  word_freq_mail:  ...  char_freq_;:  char_freq_(:  \\\n",
       "1698              0.87             0.00  ...         0.000         0.042   \n",
       "1437              0.76             0.31  ...         0.000         0.087   \n",
       "1808              0.00             0.00  ...         0.077         0.038   \n",
       "2353              0.00             0.00  ...         0.000         0.000   \n",
       "523               0.00             0.11  ...         0.000         0.057   \n",
       "\n",
       "      char_freq_[:  char_freq_!:  char_freq_$:  char_freq_#:  \\\n",
       "1698         0.000         0.000         0.085         0.000   \n",
       "1437         0.054         0.439         0.241         0.065   \n",
       "1808         0.000         0.000         0.000         0.038   \n",
       "2353         0.000         0.955         0.000         0.000   \n",
       "523          0.000         0.420         0.191         0.210   \n",
       "\n",
       "      capital_run_length_average:  capital_run_length_longest:  \\\n",
       "1698                        5.145                           33   \n",
       "1437                        3.702                           45   \n",
       "1808                        2.600                           42   \n",
       "2353                        1.500                            5   \n",
       "523                         8.026                          283   \n",
       "\n",
       "      capital_run_length_total:  spam  \n",
       "1698                        247     1  \n",
       "1437                       1070     1  \n",
       "1808                        182     1  \n",
       "2353                         24     0  \n",
       "523                        1509     1  \n",
       "\n",
       "[5 rows x 58 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Data\n",
    "cols = ['word_freq_make:',\n",
    "    'word_freq_address:',\n",
    " 'word_freq_all:',\n",
    " 'word_freq_3d:',\n",
    " 'word_freq_our:',\n",
    " 'word_freq_over:',\n",
    " 'word_freq_remove:',\n",
    " 'word_freq_internet:',\n",
    " 'word_freq_order:',\n",
    " 'word_freq_mail:',\n",
    " 'word_freq_receive:',\n",
    " 'word_freq_will:',\n",
    " 'word_freq_people:',\n",
    " 'word_freq_report:',\n",
    " 'word_freq_addresses:',\n",
    " 'word_freq_free:',\n",
    " 'word_freq_business:',\n",
    " 'word_freq_email:',\n",
    " 'word_freq_you:',\n",
    " 'word_freq_credit:',\n",
    " 'word_freq_your:',\n",
    " 'word_freq_font:',\n",
    " 'word_freq_000:',\n",
    " 'word_freq_money:',\n",
    " 'word_freq_hp:',\n",
    " 'word_freq_hpl:',\n",
    " 'word_freq_george:',\n",
    " 'word_freq_650:',\n",
    " 'word_freq_lab:',\n",
    " 'word_freq_labs:',\n",
    " 'word_freq_telnet:',\n",
    " 'word_freq_857:',\n",
    " 'word_freq_data:',\n",
    " 'word_freq_415:',\n",
    " 'word_freq_85:',\n",
    " 'word_freq_technology:',\n",
    " 'word_freq_1999:',\n",
    " 'word_freq_parts:',\n",
    " 'word_freq_pm:',\n",
    " 'word_freq_direct:',\n",
    " 'word_freq_cs:',\n",
    " 'word_freq_meeting:',\n",
    " 'word_freq_original:',\n",
    " 'word_freq_project:',\n",
    " 'word_freq_re:',\n",
    " 'word_freq_edu:',\n",
    " 'word_freq_table:',\n",
    " 'word_freq_conference:',\n",
    " 'char_freq_;:',\n",
    " 'char_freq_(:',\n",
    " 'char_freq_[:',\n",
    " 'char_freq_!:',\n",
    " 'char_freq_$:',\n",
    " 'char_freq_#:',\n",
    " 'capital_run_length_average:',\n",
    " 'capital_run_length_longest:',\n",
    " 'capital_run_length_total:', 'spam']\n",
    "\n",
    "df = pd.read_csv('spambase.data', names=cols)\n",
    "df.sample(n=5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1b0eb2ce",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data (3680, 57) (3680,)\n",
      "Test data (921, 57) (921,)\n"
     ]
    }
   ],
   "source": [
    "# Split the dataset into testing and training data\n",
    "X = df.drop(['spam'], axis=1)\n",
    "y = df['spam']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=123)\n",
    "\n",
    "print(\"Training data\", X_train.shape, y_train.shape)\n",
    "print(\"Test data\", X_test.shape, y_test.shape)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1758fe0c",
   "metadata": {},
   "source": [
    "## I will start by creating a bootstrap sample of size 1000 and then select a subset of p columns. I will vary the value of p, and report the p that results in the lowest cross-validation error. Then, I will train a decision tree classifier on the bootstrap sample (using default depth = 6)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ed10e675",
   "metadata": {},
   "outputs": [],
   "source": [
    "# find best subset p, using average CV\n",
    "n, p = X_train.shape\n",
    "scores = {i: [] for i in range(1, 57)}\n",
    "\n",
    "for i in range(1000): \n",
    "    random_p = random.randint(1, 56)\n",
    "    X_idx = random.sample([x for x in range(1, n)], 1000)\n",
    "    Y_idx = random.sample([x for x in range(1, p)], random_p)\n",
    "    \n",
    "    subsample_X = X_train.iloc[X_idx, Y_idx]\n",
    "    subsample_Y = y_train.iloc[X_idx]\n",
    "    \n",
    "    dectree = DecisionTreeClassifier()\n",
    "    model = dectree.fit(subsample_X, subsample_Y)\n",
    "    cv_scores = cross_val_score(model, subsample_X, subsample_Y, cv=10)\n",
    "    scores[random_p].append(np.mean(cv_scores))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "4fb40bc6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Subset of p = 52, that results in the best average cross validation score = 0.879\n"
     ]
    }
   ],
   "source": [
    "subset_p = []\n",
    "cv_avg = []\n",
    "\n",
    "for key, value in scores.items():\n",
    "    subset_p.append(key)\n",
    "    cv_avg.append(np.mean(np.abs(value)))\n",
    "\n",
    "\n",
    "cv_df = pd.DataFrame({'subset_p': subset_p, 'Average CV Score': cv_avg}).sort_values('Average CV Score', ascending=False)\n",
    "\n",
    "\n",
    "# get value of p with best average cv\n",
    "val = cv_df.iloc[0,:]\n",
    "print(\"Subset of p = {:.0f}, that results in the best average cross validation score = {:.3f}\".format(val[0], val[1]))\n",
    "# cv_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "cb6755c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE = 0.154\n"
     ]
    }
   ],
   "source": [
    "best_p = int(val[0])\n",
    "best_p\n",
    "\n",
    "#tree at best subset of p\n",
    "dectree = DecisionTreeClassifier(max_depth=6)\n",
    "model = dectree.fit(X_train[:best_p], y_train[:best_p])\n",
    "ypred = model.predict(X_test[:best_p])\n",
    "\n",
    "mse = mean_squared_error(y_test[:best_p], ypred)\n",
    "print('MSE = {:.3f}'.format(mse))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b85f8cc",
   "metadata": {},
   "source": [
    "## I will repeat these calculations to generate T ∈ {1, 50, 100, 150, 200, 300, 400} trees and evaluate on the training set.  I will combine the predictions from all trees and assign the final class based on a majority vote of the predictions of every tree. In case of ties, I will assign a class randomly among the ties. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "eb8f4249",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "When T = 1, MSE = 0.073, Model Score = 0.71386\n",
      "When T = 50, MSE = 0.055, Model Score = 0.92255\n",
      "When T = 100, MSE = 0.055, Model Score = 0.92473\n",
      "When T = 150, MSE = 0.058, Model Score = 0.92446\n",
      "When T = 200, MSE = 0.059, Model Score = 0.92337\n",
      "When T = 300, MSE = 0.058, Model Score = 0.92527\n",
      "When T = 400, MSE = 0.059, Model Score = 0.92310\n"
     ]
    }
   ],
   "source": [
    "T =  [1, 50, 100, 150, 200, 300, 400]\n",
    "\n",
    "for num_trees in T:\n",
    "    dtree = BaggingClassifier(DecisionTreeClassifier(max_depth=6), n_estimators=num_trees, oob_score=True) \n",
    "    model = dtree.fit(X_train, y_train)\n",
    "    ypred = model.predict(X_test) # prediction for test set \n",
    "#     print(ypred)\n",
    "    mse = mean_squared_error(ypred, y_test)\n",
    "    score = model.oob_score_\n",
    "    print('When T = {}, MSE = {:.3f}, Model Score = {:.5f}'.format(num_trees, mse, score))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad4d9e99",
   "metadata": {},
   "source": [
    "When model score < 0.5, we will classify the label to be 0, thus not predicted to be spam. For all values of T, the score is close to 1, thus predicted to be spam. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f706cef1",
   "metadata": {},
   "source": [
    "## Now I will report the training and test error, F1 score, and AUC by varying T in the range {1, 50, 100, 150, 200, 300, 400}."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8bb2f8c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "When T = 1, Training Error = 0.077, Testing Error = 0.923, F1 Score = 0.924, AUC = 0.929\n",
      "\n",
      "When T = 50, Training Error = 0.055, Testing Error = 0.945, F1 Score = 0.945, AUC = 0.948\n",
      "\n",
      "When T = 100, Training Error = 0.053, Testing Error = 0.947, F1 Score = 0.947, AUC = 0.951\n",
      "\n",
      "When T = 150, Training Error = 0.060, Testing Error = 0.940, F1 Score = 0.941, AUC = 0.942\n",
      "\n",
      "When T = 200, Training Error = 0.060, Testing Error = 0.940, F1 Score = 0.941, AUC = 0.942\n",
      "\n",
      "When T = 300, Training Error = 0.059, Testing Error = 0.941, F1 Score = 0.942, AUC = 0.944\n",
      "\n",
      "When T = 400, Training Error = 0.059, Testing Error = 0.941, F1 Score = 0.942, AUC = 0.944\n",
      "\n"
     ]
    }
   ],
   "source": [
    "T = [1, 50, 100, 150, 200, 300, 400]\n",
    "\n",
    "for num_trees in T:\n",
    "    dtree = BaggingClassifier(DecisionTreeClassifier(max_depth = 6), n_estimators=num_trees, random_state=123)\n",
    "    model = dtree.fit(X_train, y_train)\n",
    "    ypred = model.predict(X_test)\n",
    "    mse = mean_squared_error(ypred, y_test)\n",
    "    score = accuracy_score(ypred, y_test)\n",
    "    f1 = f1_score(ypred, y_test, average='weighted')\n",
    "    auc = roc_auc_score(ypred, y_test, average='weighted')\n",
    "    print('When T = {}, Training Error = {:.3f}, Testing Error = {:.3f}, F1 Score = {:.3f}, AUC = {:.3f}\\n'.format(num_trees, mse, score, f1, auc))\n",
    "    # print(classification_report(y_true=y_test, y_pred=ypred, target_names=['not spam', 'spam']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c6cce5e9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  92.94245385450597 %\n"
     ]
    }
   ],
   "source": [
    "clf = DecisionTreeClassifier(max_depth=6)\n",
    "clf.fit(X_train, y_train)\n",
    "y_pred = clf.predict(X_test)\n",
    "classification_report(y_true=y_test, y_pred=y_pred)\n",
    "\n",
    "roc_auc_score(y_pred, y_test, average='weighted')\n",
    "print('Accuracy: ', accuracy_score(y_test, y_pred)*100, '%')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa6e70a0",
   "metadata": {},
   "source": [
    "## I will use RandomForestClassifer from sklearn.ensemble library to train a Random Forest algorithm with 10, 50, and 100 decision trees and report similar metrics on both the training and testing sets, and the top 10 features having the most influence on the model. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c1c8522b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "When T = 10, MSE = 0.066\n",
      "When T = 50, MSE = 0.067\n",
      "When T = 100, MSE = 0.065\n",
      "\n",
      " Top 10 features and their Importance having the most influence on the model:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>char_freq_$:</th>\n",
       "      <td>0.143483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>char_freq_!:</th>\n",
       "      <td>0.114414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>word_freq_remove:</th>\n",
       "      <td>0.092214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>word_freq_free:</th>\n",
       "      <td>0.084814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>word_freq_your:</th>\n",
       "      <td>0.065117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>word_freq_hp:</th>\n",
       "      <td>0.059523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>capital_run_length_average:</th>\n",
       "      <td>0.059337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>capital_run_length_longest:</th>\n",
       "      <td>0.058074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>word_freq_money:</th>\n",
       "      <td>0.049315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>word_freq_our:</th>\n",
       "      <td>0.033174</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             Importance\n",
       "char_freq_$:                   0.143483\n",
       "char_freq_!:                   0.114414\n",
       "word_freq_remove:              0.092214\n",
       "word_freq_free:                0.084814\n",
       "word_freq_your:                0.065117\n",
       "word_freq_hp:                  0.059523\n",
       "capital_run_length_average:    0.059337\n",
       "capital_run_length_longest:    0.058074\n",
       "word_freq_money:               0.049315\n",
       "word_freq_our:                 0.033174"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T = [10, 50, 100]\n",
    "\n",
    "for num_trees in T:\n",
    "    rf = RandomForestClassifier(max_depth=6, random_state=123, n_estimators=num_trees)\n",
    "    modelrf = rf.fit(X_train, y_train)\n",
    "    ypred = modelrf.predict(X_test)\n",
    "    mse = mean_squared_error(ypred, y_test)\n",
    "    print('When T = {}, MSE = {:.3f}'.format(num_trees, mse))\n",
    "\n",
    "# Create dataframe for top 10 features:\n",
    "top = pd.DataFrame({'Features': X_train.columns, 'Importance': modelrf.feature_importances_}, columns=['Features', 'Importance']).sort_values('Importance', ascending=False, axis=0)\n",
    "\n",
    "print(\"\\n Top 10 features and their Importance having the most influence on the model:\")\n",
    "top[:10]\n",
    "\n",
    "# get most important features and print them\n",
    "top_df = pd.DataFrame({'Importance': modelrf.feature_importances_}, index=X_train.columns).sort_values('Importance', ascending=False, axis=0)\n",
    "top_df = top_df[:10]\n",
    "top_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "b7a6b1fb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf8AAAEWCAYAAABoup70AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA3t0lEQVR4nO3deZwcVb3//9ebsJOwJmKiSAQXBIHRDJthCQSXywVBDSACkojyQ0VcLlcvX7jcsHkFveLCRYioQUGNAWVTryBkE7IwE4YkIItABAQx7AQJhOTz++OcgaLT090zPT1L+v18PPox1VWnzvlUTXd/qs6prlZEYGZmZs1jnf4OwMzMzPqWk7+ZmVmTcfI3MzNrMk7+ZmZmTcbJ38zMrMk4+ZuZmTUZJ38z6zOSlko6sL/jaAZKfiLpaUkL+jueZibpTknj+juOIid/W6tJWl54rJb0YuH50b3UxhGSbpX0T0kzyyxvkdSel7dLaqlQ11RJL5fEfWSd8U2VdE49dQw0ksZJeqS/4wCQNFpSSFq3v2MpsTfwfuDNEbF7b1Qo6XhJd0t6XtLjkn4raVhv1F1nXEvze/t5Sc/k9+OJkmrKcb35eir3fouInSJiZm/U31uc/G2tFhFDOx/AQ8AhhXlX9FIzTwHfAb5RukDS+sA1wOXAFsBlwDV5flfOL8YdEdN6Kc4eGYBJbcAY4PtmW2BpRLzQ3RXLbZek/YCvA0dFxDDgXcCv6o6y9xyS49qW9F78GvCj/g1pAIsIP/xoigewFDgwT29AStiP5sd3gA3ysnHAI8D/A57I6x1dQ/2fBmaWzPsA8DdAhXkPAR/qoo6pwDll5q8D/AdwP/Ak6UN3y8Ly6cDfgWeB2cBOef4JwErgZWA5cF2eH8DbyrVb2P6v5Tp/Vql9YEPSwc2TwDPAbcDWFf4HpwJ3AU8DPwE2zMuWkD7AO8uul/d/S5l6xgGPFJ7PBM4Bbu3cTmAr4ArguRzT6EL5AE4GHshtfBNYp7CvTwf+CvwD+CmwWV42Oq97fP4/zs5/I7e7HNgL2B64Oe+TJ3Icm5fsh1OARfl/Nq1zP+TlhwIdOfb7ya8XYDNSQnuM9Lo6BxhSZv8cD6wAVuWYzszzPwP8hXTAei0wqmSffB64D3iwTJ2nAFdXeP1PBS4GbgSeB2YB2xaWfxd4OG9TO7BPYdlk0mv48rzuYuAd+bXyj7zeB2p5bxfm7Q6sBt5deM9/K/+/Hs+xbgRsAryYy3b+D0dR/T23N+n19kyObyJdv99ejY/aPnv+LW/3Y8CkRnwe+szfmtVpwJ5AC7Ar6YPi9MLyNwLDgTcBxwFTJL2zB+3sBCyK/M7OFuX53XEycBiwH+mD6WngfwvLfw+8HXgDsJCUbIiIKXm6szfhkBrbeyOwJeks6oQq7R9HSkrbkBLuiaQP064cDXyQlCDfwWv7/afAMYVyBwGPRURHjTF/HDiW9D/bHphLOrjYEvgz8F8l5T8CtALvJSXbT+X5E/Njf2A7YChwYcm6+5HOfD8I7JvnbZ738VxAwH+T9tW7SPtmckkdRwAfAt4K7JLbRNLupH3x78Dmuf6leZ3LgFeAtwHvIR1cfrp0R0TEj0j/h7k5pv+SdECO6QhgJOng5pclqx4G7AHsWFonMB/4oKQzJY2VtEGZMkcDZ5PeOx3k12F2G+n9tiXwc2C6pA0Lyw8hHWhuAdwO/IGUgN8EnAVcUqa9LkXEAlIi3SfPOo/0emsh7b83AWdE6hn5F+DReK237VEqvOYlvYX0nvs+MCLX2VHj+62Wz57NcnzHA/8raYvubHtNGnFE4YcfA/HB64++7wcOKiz7IKmLFNLR9yvAJoXlvwL+s0r95c78/xP4Zcm8K4DJXdQxlXTG9kx+PJHn/xkYXyg3knSGsW6ZOjYnncVtVqjznJIy1c78X+b1Z6Jdtk9KmrcCu9T4Pzix8Pwg4P48PYp01rdpfn4l8NUu6hnHmmf+pxWe/w/w+8LzQ0gfzsXt/1Dh+eeAm/L0TcDnCsveWdjW0Xnd7QrLO+et8b8olDkMuL1kPxxTeH4+cHGevgS4oEwdWwMvARsV5h0FzOiizYnAnwrPf0RKSp3Ph+btGl3YJwdU+f/9C6lX5RnSme23yT0P+TX0y5L6VwHbdFHX08CueXoycGPJ/2t5oe5hOb7NK7yuDiwzfx4p2Qp4Adi+sGwvcg9H6euphtf8qcBvKryHS99vr8ZH9c+eF4uvJVIPwJ7V3lvdfQzk8SqzRhpFOvPp9Nc8r9PT8fqx0tLltVoObFoyb1NSkuvKtyLi9JJ52wK/kbS6MG8VsLWkvwPnAoeTzkI6ywwndSn3xLKIWFFL+6SztW2AX0ranNR1e1pErOyi7ocL06/u14h4VNItwMck/YaUaL7YjZgfL0y/WOb50FrioPxrY13StpZbdw2S3gB8j3TWOYx0Bvt0SbG/F6b/WWh/G+B3ZardljQU8pikznnrVIulYBSpVwiAiFgu6UnSGebSPLtiXRHxe+D3+UK6/Uld9ffw2ln5w4WyyyU9ldt9WNK/kQ6QR5ES+aak12in0v/XExGxqvAc0v/wmRq2tdObSEMcI4CNgfbCvhMwpMK6lV7z25CSeE9U++x5MiJeKTz/J2u+duvmbn9rVo+S3tyd3pLnddpC0iYVltfqTmAXFT5xSF28d3aznoeBf4mIzQuPDSPib8AnSN3WB5K6C0fndTrbjDVqSx8oGxeev7Fkeek6XbYfESsj4syI2BF4H3Aw8MkK27JNYbp0v15G6vo/nNRl/bcK9dSrqzjKvTZe4fXJKbqY7vTfef4uEbEpaZtUplw5D5OGLcrNfwkYXvgfbBoRtQ4hvW678ut7K9K1A53KbcsaImJ1RNxEuq7h3YVFr+5TSUNJXfyPStqHdA3JEcAWEbE56cC01n3SbZJ2IyX/P5Guu3iRdC1M577bLNKFwFB+uyu957r6H3VVV1G1z54+4eRvzeoXwOmSRkgaDpxBOmMtOlPS+vmD62DSWc4aJA3JY5frAutI2lDSennxTNLZwsmSNpB0Up5/czfjvRg4V9K2uc0Rkg7Ny4aRksKTpIT+9ZJ1HyeNXRd1AJ/IsX+INK7Zo/Yl7S9pZ0lDSBdzrSRtc1c+L+nNkrYkXVRZ/DbD1aQx+C+Sxr0b6d8lbSFpm9xeZxy/AL4s6a05gX0dmFZyNla0jNTbUtzHw0i9Ps9IehNp/L5WPwImSRovaR1Jb5K0Q0Q8BtwA/I+kTfOy7fNV+LX4ea63JY/Xfx2YHxFLa1lZ0qGSPp73mfK1CfuRutY7HSRp7/xtlrNz/Q+T9scrpH21rqQzWLNHrFfkfXMw6XqGyyNicUSsBn4IXJB7Zcj79YN5tceBrSRtVqiq0nvuCuBApa/5ritpK732Fd5y77eiWj57Gs7J35rVOUAb6eK7xaTu0OJ3c/9O6qZ9lPRGPzEi7u6irmNJZxU/IHXzvkj6oCEiXiaN936S1F35KeCwPL87vku6OvsGSc+TPnD3yMt+Suo6/BvpKvp5Jev+CNgxf//56jzvi6Rx1WdIF2ldTWWV2n8jaXz+OdI46Swqf5j9nJTEHsiPV/d7RLwIXEW6CO7XVWKq1zWkq847gN/y2tfCfkwaypgNPEi6BuMLXVUSEf8kDbvckvfxnsCZpIOYZ3PdNW9LpAvVJgEX5PVn8dqZ4ieB9Xnt2xJXksaia6n3JtI1KFeRriLfnnSRZK2eJn1b4D7S//py4Jvx+q/M/px0YeVTwBjSawvSxXu/B+4lvVZXUPtwRa2uy6/Nh0nj/N8m7cdOXyN902GepOeAP5Ku5yC/t38BPJD/h6Oo8JqPiIdI16v8W97WDtLFe1D+/VZU7bOnTyhfUGBmmdKduC6PiDf3cyhNKZ8VviMijqlauOdtBPD2iPhLo9poNpKmki6aK71exQYgX/BnZgNGHgo4ntSbYmYN4m5/MxsQJH2G1GX7+4iY3d/xmK3N3O1vZmbWZHzmb2Zm1mQ85m+DwvDhw2P06NH9HYaZ2aDS3t7+RESMKJ3v5G+DwujRo2lra+vvMMzMBhVJfy03393+ZmZmTcbJ38zMrMk4+ZuZmTUZJ38zM7Mm4wv+bFBobwc17Pe/zMwGpkbdisdn/mZmZk3Gyd/MzKzJOPkPEpKmSprQgHoPl/RnSTN6u+4ybU1udBtmZladk38TUNLV//p44HMRsX/JOr12PYikj0jqAD4r6RZJO/dW3WZm1n1O/gOUpE9KWiTpDkk/y7P3lXSrpAc6ewEkDZV0k6SFkhZLOjTPH53P6C8CFgLblGnjDGBv4GJJ35Q0UdJ0SdcBN0jaRNKPJd0m6fZC3RtJ+mWOb5qk+ZJaK2zORcARwA+AjwL/qHEfnCCpTVIbLKtpv5mZWQ0iwo8B9gB2Au4BhufnWwJTgemkA7Ydgb/kZesCm+bp4cBfAAGjgdXAnlXamgm05umJwCPAlvn514Fj8vTmwL3AJsBXgB/n+bsAr3TW0UUbDwP7AJNL5rcCl9a2T8ZEuu7VDz/88KN5HvUC2sp9pvqrfgPTAcCVEfEEQEQ8pfQ9t6sjYjVwl6Stc1kBX5e0LynZvwnoXPbXiJjXzbZvjIin8vQHgA9LOiU/3xB4C7Av8L0c2yJJi6rU+XHgbGBnSaOA/xcRT0REG/DpbsZnZmZ1cvIfmAREmfkvlZQBOBoYAYyJiJWSlpKSNMALPWi7uI6Aj0XEPa8LLh2IlIuvrIi4BThA0nnAKuA80rUGZmbWDzzmPzDdBBwhaSsASVtWKLsZ8I+c+PcHtu3FOP4AfEE520t6T54/m3TQgaR3k7r+u5TLALwILAKG9WKMZmbWTT7zH4Ai4k5J5wKzJK0Cbq9Q/ArgunRRHB3A3b0YytnAd4BF+QBgKXAw6cK9n+Tu/g5gQZV6zpE0HHgr8DfgUwD5IsETI8Jd/2ZmfUj5wiuzHpM0Ezglj+FXKjc5Iib3pI3W1tZoa6tYvZmZlZDUHhFrfBvL3f7Wl2b2dwBmZuZu/6YhaT6wQcnsYyNicb11R8S4WtqIiJn1tmVmZvVz8m8SEbHH2tCGmZnVz93+ZmZmTcbJ38zMrMk4+ZuZmTUZJ38zM7Mm4+RvZmbWZJz8zczMmoyTv5mZWZPx9/xtUGhvB6l6OTNbe/ju843jM38zM7Mm4+RvZmbWZJz8ByFJUyVNaEC9h0v6s6QZvV13oY2Jki5sVP1mZladk3+TUdLV//144HMRsX/JOr42xMxsLeLkPwhI+qSkRZLukPSzPHtfSbdKeqCzF0DSUEk3SVooabGkQ/P80fmM/iJgIbBNmTbOAPYGLpb0zXyGPl3SdcANkjaR9GNJt0m6vVD3RpJ+meObJmm+pDV+O7qH232CpDZJbbCsN6o0MzN8tf+AJ2kn4DRgbEQ8IWlL4NvASFKy3gG4FrgSWAF8JCKekzQcmCfp2lzVO4FJEfG5cu1ExFmSDgBOiYg2SROBvYBdIuIpSV8Hbo6IT0naHFgg6Y/A/wf8MyJ2kbQL6eCiO9t3KXBxRLSViWkKMCWVa/V1v2ZmvcTJf+A7ALgyIp4AyIkY4OqIWA3cJWnrXFbA1yXtC6wG3gR0LvtrRMzrZts3RsRTefoDwIclnZKfbwi8BdgX+F6ObZGkRd1pICI+3c2YzMysTk7+A5+Acme9L5WUATgaGAGMiYiVkpaSkjTACz1ou7iOgI9FxD2vCy4diNR8Vh4RU4GpPYjFzMx6icf8B76bgCMkbQWQu/27shnwj5z49we27cU4/gB8QTnbS3pPnj+bdNCBpHcDu/Rim2Zm1gA+8x/gIuJOSecCsyStAm6vUPwK4Lp0gRwdwN29GMrZwHeARfkAYClwMPAD4Ce5u78DWFCpEkkfBnaMiG/k512O+ReNGQNtFUuYmVmtFL5/ovUiSTPJFw32Zr2tra3R5uxvZtYtktojYo1vYLnb38zMrMm4278JSZoPbFAy+9iIWFxv3RExrtFtmJlZfZz8m1BE7LE2tGFmZj3jbn8zM7Mm4+RvZmbWZJz8zczMmoyTv5mZWZNx8jczM2syTv5mZmZNxl/1s0GhvR2k6uXMLPHNW60Sn/mbmZk1GSd/MzOzJuPk3wckTZR0YYXlIyTNl3S7pH36MjYzM2s+Tv4NIGlIN1cZD9wdEe+JiDl11lWVJF/rYWbWxJz8S0j6qqST8/QFkm7O0+MlXS7pKEmLJS2RdF5hveWSzso/aLOXpEmS7pU0Cxhbob0W4HzgIEkdkjYqU9cxkhbk5Zd0HhAU25D0wyq9C1MlfVvSDOA8SdtL+j9J7ZLmSNqhUO4HkmZIekDSfpJ+LOnPkqYW6ltjP0j6rKTzC2UmSvp+ni67DWZm1vec/Nc0G+jsem8FhkpaD9gbuA84DzgAaAF2k3RYLrsJsCT/oM39wJmkpP9+YMeuGouIDuAMYFpEtETEiyV1PQkcCYyNiBZgFXC0pJG1tlHwDuDAiPg3YArwhYgYA5wCXFQot0Xexi8D1wEXADsBO0tqkTSqi/1wJfDRQj1HAtMkvavcNgBIulTSGr81nZedIKlNUhssq2HzzMysFu7+XVM7MEbSMOAlYCHpIGAfUiKcGRHLACRdAewLXE1KaFflOvYoKTeNlHhrVaxrPDAGuE3pu24bAf/oYRvTI2KVpKHA+4Dpeu37c8Wf370uIkLSYuDxzp/hlXQnMBrYttx+iIirc2/BnqQDpXcCtwCf72IbiIhPdxVsREwhHaQgtfqLS2ZmvcTJv0RErJS0FJgE3AosAvYHtgceIiWxclZExKpiVXWEUaxLwGURcWqxQD7T7m4bL+S/6wDP5LPwcl7Kf1cXpjufrwu8UqGNacARwN3Ab/JBRNltMDOz/uFu//Jmk7rCZwNzgBOBDmAesJ+k4XnM+ihgVpn15wPjJG2VhwwOryOWm4AJkt4AIGlLSdvW00ZEPAc8KOnwXKck7dqNmObT9X74NXBYnjetyjaYmVk/cPIvbw4wEpgbEY8DK4A5EfEYcCowA7gDWBgR15SunMtNBuYCfyQNHfRIRNwFnA7cIGkRcCMwshfaOBo4XtIdwJ3Aod2Iqcv9EBFPA3cB20bEgkrbAJXH/M3MrDEUvgfkWkHSRKA1Ik7q71gaobW1Ndra2vo7DDOzQUVSe0SscYLlM38zM7Mm4wv++pCk01hzbH56RJxbb90RMRWY2sg2zMxs7eBufxsU3O1vZtZ97vY3MzMzwMnfzMys6Tj5m5mZNRknfzMzsybj5G9mZtZknPzNzMyajJO/mZlZk/FNfmxQaG+H13592Gzt5tuvWKP5zN/MzKzJOPmbmZk1GSf/QUTSREkXVlg+QtJ8SbdL2qdBMTS8DTMzayyP+Q9gkoZExKpurDIeuDsijuuFuvqzDTMzayCf+TeIpK9KOjlPXyDp5jw9XtLlko6StFjSEknnFdZbLuksSfOBvSRNknSvpFnA2ArttQDnAwdJ6pC0UZm6jpG0IC+/RNKQvO6rbUj6YVe9C3W28QFJcyUtlDRd0tAa9uEJktoktcGyWna7mZnVwMm/cWYDnd3ircBQSesBewP3AecBBwAtwG6SDstlNwGWRMQewP3AmaSk/35gx64ai4gO4AxgWkS0RMSLJXU9CRwJjI2IFmAVcLSkkX3QxnDgdODAiHgv0AZ8BSAfOHy4i/amRERr+kWqEV2FZWZm3eRu/8ZpB8ZIGga8BCwkHQTsA1wHzIyIZQCSrgD2Ba4mJcyrch17lJSbBryjGzEU6xoPjAFuU/rO3EbAP/qojT1JBxW35PnrA3MBIuKMbrRlZma9wMm/QSJipaSlwCTgVmARsD+wPfAQKUmWs6Jk3Lyeb/wW6xJwWUScWiyQexwa3cYhwI0RcVQd7ZiZWS9xt39jzQZOyX/nACcCHcA8YD9Jw/OY+FHArDLrzwfGSdoqDxkcXkcsNwETJL0BQNKWkrbtozbmAWMlvS3P31hSd3oXzMysFzn5N9YcYCQwNyIeB1YAcyLiMeBUYAZwB7AwIq4pXTmXm0zqIv8jaeigRyLiLtK4+w2SFgE3AiP7qI1lwETgF3n+PGAHqDzmb2ZmjaHwfSStQNJEoDUiTurvWIpaW1ujra2tv8MwMxtUJLWni6Zfz2f+ZmZmTcYX/A1Ckk5jzbH56RFxbr11R8RUYGoj2zAzs/7lbn8bFNztb2bWfe72NzMzM8DJ38zMrOk4+ZuZmTUZJ38zM7MmU1Pyl7S9pA3y9DhJJ0vavKGRmZmZWUPUeuZ/FbAq3571R8BbgZ83LCozMzNrmFqT/+qIeAX4CPCdiPgy6ba1ZmZmNsjUepOflZKOAo4DDsnz1mtMSGZram+H9GvAZj3jW5qYvabWM/9JwF7AuRHxoKS3Apc3LiwzMzNrlJrv8CdpI+AtEXFPY0MyW5PUGuA7/FnP+czfmlFdd/iTdAjpd+j/Lz9vkXRtr0ZoZUmaKOnCCstHSJov6XZJ+/RlbGZmNjjV2u0/GdgdeAYgIjpIV/xbL5M0pJurjAfujoj3RMScOuvqM5L8o1JmZv2k1uT/SkQ8WzLPnWglJH1V0sl5+gJJN+fp8ZIul3SUpMWSlkg6r7DecklnSZoP7CVpkqR7Jc0CxlZorwU4HzhIUoekjcrUdYykBXn5JZ0HBMU2JP2wq94FScMkPShpvfx8U0lLJa2Xe4DmSVok6TeStshlZkpqzdPDJS3N0xMlTZd0HXBDDfvzBEltktpgWbXiZmZWo1qT/xJJnwCGSHq7pO8DtzYwrsFqNtDZ9d4KDM1Jc2/gPuA84ACgBdhN0mG57CbAkojYA7gfOJOU9N8P7NhVY7kH5gxgWkS0RMSLJXU9CRwJjI2IFmAVcLSkkd1o43lgJvCvedbHgasiYiXwU+BrEbELsBj4r2o7iHTh6HERcYCkUZJ+V6HtKRHRmsarRtRQtZmZ1aLW5P8FYCfgJdLNfZ4FvtSgmAazdmCMpGGkfTWXdBCwD2nIZGZELMv3TLgC2Devt4p0IyWAPQrlXgamdTOGYl3jgTHAbZI68vPtetDGpaRvfJD//kTSZsDmETErz7+ssD2V3BgRTwFExKMRcVBtm2VmZr2l6rhr7ia+NiIOBE5rfEiDV0SszF3ck0g9I4uA/YHtgYdIibicFRGxqlhVHWEU6xJwWUScWiyQexxqbiMibpE0WtJ+wJCIWJKTf1de4bUDyw1Llr1Qa7tmZtYYVc/8cyL5Z5UPe3vNbOCU/HcOcCLpmxLzgP3yGPgQ4ChgVpn15wPjJG2VhwwOryOWm4AJkt4AIGlLSdv2sI2fAr8AfgKQrwF5uvANg2ML27OU1w50JtQRv5mZNUCtV1yvABZLupHCmVtEnNyQqAa3OaQekrkR8YKkFcCciHhM0qnADNIZ+e8i4prSlXO5yaQhg8eAhUCPrtqPiLsknQ7cIGkdYCXw+YiY14M2rgDOIR0AdDoOuFjSxsADvDY08C3gV5KOBW7uqkJJo4BL3fVvZta3arrJj6Tjys2PiMt6PSLrc5ImAq0RcVKFMhOAQyPi2D4LrKC1tTXa2nyTHzOz7ujqJj81nfk7yTe3/O2OfwF8hm5mthaoKflLepAyF4hFxHa9HpGVJek01hybnx4R59Zbd0RMBaZWaOML9bZhZmYDR61j/sUugw1JCWLL3g/HupKTfN2Jvr/bMDOz/lfT9/wj4snC428R8R3SzWrMzMxskKm12/+9hafrkHoChjUkIjMzM2uoWrv9/6cw/QrwIHBE74djZmZmjVZr8j8+Ih4ozpDkX/UzMzMbhGq9t/+VNc4zMzOzAa7imb+kHUg/6LOZpI8WFm3KmvdsNzMzs0GgWrf/O4GDgc2BQwrznwc+06CYzMzMrIFqvb3vXhExtw/iMStLag3w7X0tqeFjy8yo8/a+wO2SPk8aAni1uz8iPtVL8ZmZmVkfqfWCv58BbwQ+SPrZ1jeTuv7NzMxskKk1+b8tIv4TeCH/yM+/Ajs3LizrLkkTJV1YYfkISfMl3S5pnwbFMDX/+p+ZmQ1gtSb/lfnvM5LeDWwGjG5IRFYTSUO6ucp44O6IeE9EzKmzLjMzG8RqTf5TJG0B/CdwLXAXcH7DolrLSfqqpJPz9AWSbs7T4yVdLukoSYslLZF0XmG95ZLOkjQf2EvSJEn3SpoFjK3QXgvp/3WQpA5JG5Wp6xhJC/LySzoPCIptSPphpd6FbF9Jt0p6oLMXQNI4SbMl/UbSXZIullT1tSfpBEltktpgWbXiZmZWo1p/2OfSiHg6ImZFxHYR8YaIuLjRwa3FZgOdXe+twFBJ6wF7A/cB55F+OKkF2E3SYbnsJsCSiNgDuB84k5T03w/s2FVjEdEBnAFMi4iWiHixpK4ngSOBsRHRAqwCjpY0stY2Ckbm7TgY+EZh/u7Av5GGi7YHPgog6VJJa1yJmuOeEhGt6UrVETU0bWZmtagp+UvaWtKPJP0+P99R0vGNDW2t1g6MkTQMeAmYSzoI2Ad4BpgZEcsi4hXgCmDfvN4q4Ko8vUeh3MvAtG7GUKxrPDAGuE1SR36+XQ/buDoiVkfEXcDWhfkLIuKBiFgF/IJ0gEBEfDoi/B0+M7M+VGu3/1TgD8Co/Pxe4EsNiKcpRMRKYCkwCbgVmAPsTzojfqjCqity8ny1qjrCKNYl4LLcK9ASEe+MiMk9bOOlwrQK06X1+JvaZmb9pNbkPzwifgWsBshnpKsqr2JVzAZOyX/nACcCHcA8YD9Jw/O4+1Gkr1eWmg+Mk7RVHjI4vI5YbgImSHoDgKQtJW3by23sLumteaz/SOBPddRlZmZ1qPUmPy9I2op8tiZpT+DZhkXVHOYApwFzI+IFSSuAORHxmKRTgRmkM+ffRcQ1pSvncpNJQwaPAQuBHl21HxF3SToduCEn55XA5yNiXm+1kev4BmnMfzbwG0hj/sDF1br+x4yBNg8OmJn1ilpv7/te4PvAu4ElpKuvJkTEosaGZwOJpIlAa0Sc1M31xgGnRMTBPW27tbU12pz9zcy6pUe395X0loh4KCIWStqP9EM/Au7J49ZmZmY2yFTr9r8aeG+enhYRH2tsOFYvSaex5tj89Ig4t966I2IqMLW7bUTETGBmve2bmVnvqJb8i1drb9fIQKx35ARcd6Lv7zbMzKxxql3tH11Mm5mZ2SBV7cx/V0nPkXoANsrT5OcREZs2NDozMzPrdRWTf0T4B1/MzMzWMrXe5MfMzMzWEk7+ZmZmTcbJ38zMrMnUentfs37V3g5S9XLWHGq4MamZVeAzfzMzsybj5G9mZtZkBnTylzRK0pV5ukXSQTWsM07S9Y2Pbo12J0s6pQH1fknSxoXny3u7DTMzay4DOvlHxKMRMSE/bQGqJv/ukDQYrnn4ErBxtUJ9ZZDsMzMzq6ChyV/SJyUtknSHpJ9JOkTSfEm3S/qjpK1zucl5+c2S7pP0mTx/tKQlktYHzgKOlNQh6UhJu0u6Ndd1q6R31hjTZElTJN0A/FTSREkXFpZfn3+CFknLJZ2b45/XGW8NbWwv6f8ktUuaI2mHPH+qpO/leB+QNCHPX0fSRZLuzO3/TtIESScDo4AZkmYU6q8ppnL7O7e1VNLmhXJ/yctGSLpK0m35MbaLfTY6b9fC/Hhfpe3Iy8ZImpX3yR8kjaxlX5qZWQNEREMewE7APcDw/HxLYAtA+fmngf/J05OBO4CNgOHAw6SkNxpYkstMBC4s1L8psG6ePhC4Kk+PA66vENdkoB3YqIt6rwfG5ekADsnT5wOnV6n3lDx9E/D2PL0HcHOengpMJx107Qj8Jc+fAPwuz38j8DQwIS9b2rkPexBTV/v7u8CkQnx/zNM/B/bO028B/tzFPtsY2DBPvx1oq7QdwHrArcCIXO5I4Md5+kTgxC7iPwFoS4+3RLrG2w8/wsxq1Pn5XPpoZBfuAcCVEfEEQEQ8JWlnYFo+61sfeLBQ/pqIeBF4MZ/l7g50VKh/M+AySW8HgpRganVtbqual0kHA5CS3/urrSBpKPA+YLpe+27aBoUiV0fEauCuwln73qSfxF0N/L14ll9nTG+m/P6eBpwB/AT4eH4O6SBqx0Lcm0oalqeL+2w94EJJLcAq4B1VtuOdwLuBG3PdQ4DHACLi4q6Cj4gpwBQAqTUqbKeZmXVDI5O/YI1fAvw+8O2IuDZ3rU8uLCstW+3D/mxgRkR8RNJouvd78S8Upl/h9cMfGxamV+YjJ0hJrpb9tQ7wTES0dLH8pcK0Sv7WojsxdbW/5wJvkzQCOAw4pxD7XqUHRjlhF/fZl4HHgV3zOiuqbIeAOyNirwqxmplZH2nkmP9NwBGStgKQtCXpbP1veflxJeUPlbRhLj8OuK1k+fPAsMLzYl0T64hzKdCSx6u3IfU49FhEPAc8KOlwACW7VlntT8DHcgxbk7a/U+l2d0fZ/Z0PHn4DfJvUtf9kXnQDcFJnuXxm31W9j+Uz/GNJZ/KVtuMeYISkvXK960naqYfbZGZmdWpY8o+IO4FzgVmS7iAlmsmk7vA5wBMlqywAfgvMA86OiEdLls8gdUl3SDqSNN7935Ju4bXk0xO3kLrDFwPfAhbWUVeno4Hj83bfCRxapfxVwCPAEuASYD7wbF42Bfh9laGArkym6/09DTiG17r8AU4GWpUu0ryLNB5fzkXAcZLmkbr8O3sFym5HRLxMGvs/L++TDtLQCJJOlNRVO2Zm1gB6rQe5H4OQJgPLI+Jb/R1Lf5E0NCKW556PBcDYiPh7f8fVXY3ajtbW1mhra6s/QDOzJiKpPSJaS+f7O9sDx/X563frk3o+Bl3iz9aW7TAzW2sNiOQfEZN7u05Jk4Avlsy+JSI+X2e9pwGHl8yeHhHn1lNvRIwbaDH1RD3bYWZmfWNAdPubVeNufzOz7uuq239A397XzMzMep+Tv5mZWZNx8jczM2syTv5mZmZNxsnfzMysyTj5m5mZNRknfzMzsyYzIG7yY1ZNezuoO799aGsd35LErPf4zN/MzKzJOPmbmZk1mUGV/CWNknRlnm6RdFAN64yTdH3jo1uj3cmSTmlAvV+StHHh+fL+jqk7JB0macf+jMHMrNkNquQfEY9GxIT8tAWomvy7Q9JguAbiS8DG1QoNYIcBTv5mZv2oT5O/pE9KWiTpDkk/k3SIpPmSbpf0R0lb53KT8/KbJd0n6TN5/mhJSyStD5wFHCmpQ9KRknaXdGuu61ZJ76wxpsmSpki6AfippImSLiwsv17SuDy9XNK5Of55nfHW0Mb2kv5PUrukOZJ2yPOnSvpejvcBSRPy/HUkXSTpztz+7yRNkHQyMAqYIWlGof6exNSSyy+S9BtJW+T5MyWdJ2mBpHsl7ZPnbyzpV7n8tPx/a83LPiBprqSFkqZLGprnf0PSXXmdb0l6H/Bh4Jv5/7Z9lRhPkNQmqQ2W1bJZZmZWgz5L/pJ2Ak4DDoiIXUk/t/snYM+IeA/wS+CrhVV2Af4V2As4Q9KozgUR8TJwBjAtIloiYhpwN7BvrusM4OvdCG8McGhEfKJKuU2AeTn+2cBnaqx/CvCFiBgDnAJcVFg2EtgbOBj4Rp73UWA0sDPwadI+ICK+BzwK7B8R+9cZ00+Br0XELsBi4L8Ky9aNiN1JvQyd8z8HPJ3Ln03aZ0gaDpwOHBgR7wXagK9I2hL4CLBTXueciLgVuBb49/x/u1/SiZJOLBdgREyJiNb0i1QjatwsMzOrpi+7uQ8AroyIJwAi4ilJOwPTJI0E1gceLJS/JiJeBF7MZ7m7Ax0V6t8MuEzS24EA1utGbNfmtqp5Gei8fqAdeH+1FfJZ8PuA6Xrtu2obFIpcHRGrgbsKZ+17A9Pz/L8Xz/J7KabNgM0jYlaedRkwvVDk14X6Rhdi+i5ARCyRtCjP35PUjX9L3r71gbnAc8AK4FJJvy3E+DoRcXG1eM3MrHf1ZfIXKSkXfR/4dkRcm7vWJxeWlZat9i3fs4EZEfERSaOBmd2I7YXC9Cu8vkdkw8L0yohXv228itr23zrAMxHR0sXylwrTKvlbi57EVE1nTMX6uopJwI0RcdQaC6TdgfHAx4GTSAeAZmbWz/pyzP8m4AhJWwHkbuHNgL/l5ceVlD9U0oa5/DjgtpLlzwPDCs+LdU2sI86lQEsed9+G1OPQYxHxHPCgpMMBlOxaZbU/AR/LMWxN2v5Opdvdk5ieBZ7uHM8HjgVmVVilM6YjAJSu1t85z58HjJX0trxsY0nvyD0em0XE70jDBy29Fb+ZmdWnz5J/RNwJnAvMknQH8G3Smf50SXOAJ0pWWQD8lpRczo6IR0uWzwB27LzgDzgf+G9JtwBD6gj1FtLww2LgW8DCOurqdDRwfN7uO4FDq5S/CngEWAJcAswHns3LpgC/rzIUUIvjSBfeLSIl5rOqlL8IGJHLfw1YBDwbEctIB1u/yMvmATuQEvz1ed4s4Mu5nl8C/54vzNy+0pi/mZk1hmIA3jNT0mRgeUR8q79j6S+ShkbE8tzzsQAYGxF/78d4hgDrRcSKfJX+TcA78sWXDdfa2hptbW190ZSZ2VpDUnu6aPr1BsP32pvV9ZI2J11Ad3Z/Jv5sY9JXDNcjjfN/tq8Sv5mZ9a4BmfwjYnJv1ylpEunrhUW3RMTn66z3NODwktnTI+LceuqNiHEDKaaIeB5Y4+jRzMwGnwHZ7W9Wyt3+Zmbd11W3/6C6va+ZmZnVz8nfzMysyTj5m5mZNRknfzMzsybj5G9mZtZknPzNzMyajJO/mZlZkxmQN/kxK9XeDurObx1av/MtRMwGLp/5m5mZNRknfzMzsybj5N8PJE2UdGGF5SMkzc8/e7tPX8ZmZmZrP4/59wFJQyJiVTdWGQ/cHRHH9UJdZmZmr+Mz/yokfVXSyXn6Akk35+nxki6XdJSkxZKWSDqvsN5ySWdJmg/sJWmSpHslzQLGVmivBTgfOEhSh6SNytR1jKQFefklkobkdV9tQ9IPq/QuTJX0A0kzJD0gaT9JP5b0Z0lTC+Uqbd+5ku6QNE/S1nn+CElXSbotP8ZKWkfSfZJG5DLrSPqLpOFV9v0JktoktcGySkXNzKwbnPyrmw10dr23AkPzb9rvDdwHnAccALQAu0k6LJfdBFgSEXsA9wNnkpL++4Edu2osIjqAM4BpEdESES+W1PUkcCQwNiJagFXA0ZJG1tpGwRY59i8D1wEXADsBO0tqkTSqyvbNi4hd8z76TJ7/XeCCiNgN+BhwaUSsBi4Hjs5lDgTuANaX9LsK+2JKRLSmX6QaUcPmmJlZLZz8q2sHxkgaBrwEzCUdBOwDPAPMjIhlEfEKcAWwb15vFXBVnt6jUO5lYFo3YyjWNR4YA9wmqSM/366HbVwX6TedFwOPR8TinKjvBEYDu1XYvpeB6/N0ey4PKbFfmGO7Ftg077sfA5/MZT4F/CQiHo2Ig7qzI8zMrH4e868iIlZKWgpMAm4FFgH7A9sDD5EScTkrSsbm6/nWc7EuAZdFxKnFAvmMvLttvJT/ri5Mdz5fF3ilwror84EDpIOTztfSOsBeucei6HlJj0s6gHSgcjRmZtYvfOZfm9nAKfnvHOBEoAOYB+wnaXgedz8KmFVm/fnAOElb5SGDw+uI5SZggqQ3AEjaUtK2vdxGp/nUtn1FNwAndT7J1zB0upTU/f8rX7RoZtZ/nPxrMwcYCcyNiMeBFcCciHgMOBWYQRrDXhgR15SunMtNJg0Z/BFY2NNAIuIu4HTgBkmLgBuBkb3ZRqGtmravxMlAq6RFku4iHSh1uhYYCvwEQNKoSmP+ZmbWGArfg3OtJGki0BoRJ1Ur21cktZIuBuz2vQtaW1ujra2tAVGZma29JLWni6Zfz2P+1ick/QfwWTzWb2bW73zm348kncaaY/PTI+LcwdRGX/CZv5lZ93V15u/kb4OCk7+ZWfd1lfx9wZ+ZmVmTcfI3MzNrMk7+ZmZmTcbJ38zMrMk4+ZuZmTUZJ38zM7Mm45v82KDQ3g5Sf0cx8Pmbu2ZWC5/5m5mZNRknfzMzsybj5G9mZtZknPwHKUkTJV1YYfkISfMl3S6p27+iZ2Zmay8n/0FC0pBurjIeuDsi3hMRc+qsq9cNhBjMzJqVk38fkPRVSSfn6Qsk3Zynx0u6XNJRkhZLWiLpvMJ6yyWdJWk+sJekSZLulTQLGFuhvRbgfOAgSR2SNipT1zGSFuTll3Qm42Ibkn5YpXdhW0k3SVqU/74lz58qaUJxO/LfcZJmSPo5sLiG/XaCpDZJbbCsWnEzM6uRk3/fmA10dr23AkMlrQfsDdwHnAccALQAu0k6LJfdBFgSEXsA9wNnkpL++4Edu2osIjqAM4BpEdESES+W1PUkcCQwNiJagFXA0ZJG1tpGdiHw04jYBbgC+F4N+2J34LSI2BFAUkeF7ZgSEa3pF6lG1FC1mZnVwsm/b7QDYyQNA14C5pIOAvYBngFmRsSyiHiFlET3zeutAq7K03sUyr0MTOtmDMW6xgNjgNty8h0PbNeDNvYCfp6nf0Y6mKlmQUQ82PkkH3yYmVkf8k1++kBErJS0FJgE3AosAvYHtgceIiXiclZExKpiVXWEUaxLwGURcWqxQO5xqKeNznVfIR9YShKwfqHMC3XUb2ZmvcBn/n1nNnBK/jsHOBHoAOYB+0kansfdjwJmlVl/PjBO0lZ5yODwOmK5CZgg6Q0AkraUtG0P2rgV+HiePhr4U55eymsHNIcC69URq5mZ9TIn/74zBxgJzI2Ix4EVwJyIeAw4FZgB3AEsjIhrSlfO5SaThgz+CCzsaSARcRdwOnCDpEXAjcDIHrRxMjAp13Es8MU8/4ekA5oFpKGELs/2K435F40Zk25d60flh5lZLRT+xLAuSJoItEbESf0dS2tra7S1tfV3GGZmg4qk9nTR9Ov5zN/MzKzJ+IK/QU7Saaw5Nj89Is6tt+6ImApMbWQbZmbW99ztb4OCu/3NzLrP3f5mZmYG+MzfBglJzwP39Hcc3TAceKK/g+imwRbzYIsXBl/MjrfxGh3zthGxxi1SPeZvg8U95bquBipJbYMpXhh8MQ+2eGHwxex4G6+/Yna3v5mZWZNx8jczM2syTv42WEzp7wC6abDFC4Mv5sEWLwy+mB1v4/VLzL7gz8zMrMn4zN/MzKzJOPmbmZk1GSd/61eSPiTpHkl/kfQfZZZL0vfy8kWS3lvrugMtZknbSJoh6c+S7pT0xTVrHzjxFpYPkXS7pOv7It56Y5a0uaQrJd2d9/VeAzzeL+fXwxJJv5C04QCIdwdJcyW9JOmU7qw70GIewO+7LvdxXt7Y911E+OFHvzyAIcD9wHbA+qSfNN6xpMxBwO8BAXsC82tddwDGPBJ4b54eBtzb6Jjribew/CvAz4HrB/rrIi+7DPh0nl4f2Hygxgu8CXgQ2Cg//xUwcQDE+wZgN+Bc4JTurDsAYx6o77uy8RaWN/R95zN/60+7A3+JiAci4mXgl8ChJWUOBX4ayTxgc0kja1x3QMUcEY9FxEKAiHge+DPpw39Axgsg6c3AvwKXNjjOXolZ0qbAvsCPACLi5Yh4ZqDGm5etC2wkaV1gY+DR/o43Iv4REbcBK7u77kCLeaC+7yrs4z553zn5W396E/Bw4fkjrPmm7KpMLes2Qj0xv0rSaOA9wPzeD7F7sVQp8x3gq8DqBsVXTj0xbwcsA36Su0wvlbRJI4OtEEvVMhHxN+BbwEPAY8CzEXFDA2PtMpY+WLcevdLuAHvfVfIdGvy+c/K3/qQy80q/e9pVmVrWbYR6Yk4LpaHAVcCXIuK5XoytnB7HK+lg4B8R0d77YVVUzz5eF3gv8IOIeA/wAtDocel69vEWpDPCtwKjgE0kHdPL8ZWq570zkN93lSsYeO+78iv20fvOyd/60yPANoXnb2bNLs+uytSybiPUEzOS1iN9AF0REb9uYJxVY6mhzFjgw5KWkrotD5B0eeNCrRpPLWUeAR6JiM4zuytJBwONVE+8BwIPRsSyiFgJ/Bp4XwNjrRRLo9etR13tDtD3XVf65H3n5G/96Tbg7ZLeKml94OPAtSVlrgU+ma+W3pPULfpYjesOqJgliTQW/eeI+HYfxFpXvBFxakS8OSJG5/VujohGn5XWG/PfgYclvTOXGw/cNVDjJXX37ylp4/z6GE8ak+7veBuxbj163O4Aft+V1Wfvu0ZcReiHH7U+SFdB30u6Mva0PO9E4MQ8LeB/8/LFQGuldQdyzMDepK6/RUBHfhw0UOMtqWMcfXS1fy+8LlqAtryfrwa2GODxngncDSwBfgZsMADifSPp7PU54Jk8vWlX6w6Q10TZmAfw+67LfVyoo2HvO9/e18zMrMm429/MzKzJOPmbmZk1GSd/MzOzJuPkb2Zm1mSc/M3MzJqMk7+ZrXUkLe/j9kZL+kRftmlWDyd/M7M65B/kGQ04+dugsW5/B2Bm1iiSxpFuovM46eY/vybdZOeLwEbAYRFxv6SpwApgJ2Br4CsRcb2kDYEfAK3AK3n+DEkTSb+6tiGwCenX+N4lqYP0k8K/Id2wp/NHhU6KiFtzPJOBJ4B3A+3AMRERknYDvpvXeYl0t79/At8g3exlA+B/I+KS3txH1pyc/M1sbbcr8C7gKeAB4NKI2F3SF4EvAF/K5UYD+wHbAzMkvQ34PEBE7CxpB+AGSe/I5fcCdomIp3JSPyUiDgaQtDHw/ohYIentwC9IBxCQflVuJ9K93m8BxkpaAEwDjoyI2/JPE78IHE+6FfBukjYAbpF0Q0Q82Ot7yZqKk7+Zre1ui3QffSTdD3T+ZO5iYP9CuV9FxGrgPkkPADuQbg37fYCIuFvSX4HO5H9jRDzVRZvrARdKagFWFdYBWBARj+R4OkgHHc8Cj0X6fXci/+qcpA8Au0iakNfdDHg74ORvdXHyN7O13UuF6dWF56t5/Wdg6b3Ou/rp6E4vVFj2ZdJQw66ka6tWdBHPqhyDyrRPnv+FiPhDhbbMus0X/JmZJYdLWkfS9sB2wD3AbOBogNzd/5Y8v9TzwLDC881IZ/KrgWOBIVXavhsYlcf9kTQsX0j4B+Cz+SdpkfQOSZtUqMesJj7zNzNL7gFmkS74OzGP118EXCxpMemCv4kR8VL6ldjXWQS8IukOYCpwEXCVpMOBGVTuJSAiXpZ0JPB9SRuRxvsPBC4lDQsszD9Nuww4rBe21Zqcf9XPzJpevtr/+oi4sr9jMesL7vY3MzNrMj7zNzMzazI+8zczM2syTv5mZmZNxsnfzMysyTj5m5mZNRknfzMzsybz/wPDNEn2FUnHMQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot the features and importance \n",
    "top_df.plot(kind='barh', color='b', legend=None)\n",
    "plt.ylabel('Features')\n",
    "plt.xlabel('Importance')\n",
    "plt.gca().invert_yaxis()\n",
    "plt.title('Top 10 Features by Importance for Spam Detection')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ea70586",
   "metadata": {},
   "source": [
    "From this output we can see that based on the model, the top 5 features in an email that are best at predicting spam are the frequency of characters \"$\", and \"!\", and the words \"remove\", \"free\" and \"your\". If I were working at a company sending promotional emails, I would consider limiting these characters/phrases to where it is only necessary, to try to avoid the e-mail going to the spam inbox of our e-mail list."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}