Term_deposit_II.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import metrics\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.metrics import make_scorer, accuracy_score\n",
    "from sklearn.metrics import precision_score, recall_score, f1_score, cohen_kappa_score, average_precision_score\n",
    "from sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold, GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.svm import SVC\n",
    "import xgboost as xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = pd.read_csv('C:\\\\df_data.csv')\n",
    "df = pd.DataFrame(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 41176 entries, 0 to 41175\n",
      "Data columns (total 37 columns):\n",
      " #   Column                Non-Null Count  Dtype  \n",
      "---  ------                --------------  -----  \n",
      " 0   age                   41176 non-null  float64\n",
      " 1   campaign              41176 non-null  float64\n",
      " 2   previous              41176 non-null  float64\n",
      " 3   cons.conf.idx         41176 non-null  float64\n",
      " 4   euribor3m             41176 non-null  float64\n",
      " 5   target                41176 non-null  int64  \n",
      " 6   blue_collar           41176 non-null  int64  \n",
      " 7   student               41176 non-null  int64  \n",
      " 8   retiree               41176 non-null  int64  \n",
      " 9   unemployed            41176 non-null  int64  \n",
      " 10  illiterate            41176 non-null  int64  \n",
      " 11  unknown_edu           41176 non-null  int64  \n",
      " 12  university            41176 non-null  int64  \n",
      " 13  basic4                41176 non-null  int64  \n",
      " 14  basic_other           41176 non-null  int64  \n",
      " 15  single                41176 non-null  int64  \n",
      " 16  no_default            41176 non-null  int64  \n",
      " 17  age_retiree           41176 non-null  float64\n",
      " 18  age_student           41176 non-null  float64\n",
      " 19  age_basic4            41176 non-null  float64\n",
      " 20  age_illiterate        41176 non-null  float64\n",
      " 21  poutcome_nonexistent  41176 non-null  int64  \n",
      " 22  poutcome_success      41176 non-null  int64  \n",
      " 23  contact_telephone     41176 non-null  int64  \n",
      " 24  month_aug             41176 non-null  int64  \n",
      " 25  month_dec             41176 non-null  int64  \n",
      " 26  month_jul             41176 non-null  int64  \n",
      " 27  month_jun             41176 non-null  int64  \n",
      " 28  month_mar             41176 non-null  int64  \n",
      " 29  month_may             41176 non-null  int64  \n",
      " 30  month_nov             41176 non-null  int64  \n",
      " 31  month_oct             41176 non-null  int64  \n",
      " 32  month_sep             41176 non-null  int64  \n",
      " 33  day_of_week_mon       41176 non-null  int64  \n",
      " 34  day_of_week_thu       41176 non-null  int64  \n",
      " 35  day_of_week_tue       41176 non-null  int64  \n",
      " 36  day_of_week_wed       41176 non-null  int64  \n",
      "dtypes: float64(9), int64(28)\n",
      "memory usage: 11.6 MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The data before us is very imbalanced, with only 12.7% of the positive class. The choice of scoring metric on which to train the machine learning algorithm is therefore extremely important. The typical accuracy scorer is not the best choice with such an imbalanced dataset because one could get an apparently high accuracy score of approximately 87% if the model just simply picks the negative class. This is obviously unhelpful to the goal of predicting the positive class.\n",
    "\n",
    "#### The F1 score tends to be the better metric for imbalanced datasets as it evaluates both the precision and recall rates, so it is focused on how good the model is at predicting the positive (minority in this case) class. Another good metric is Cohen's Kappa, which takes into account how much agreement would be expected by chance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#f1_scorer = make_scorer(f1_score)\n",
    "# Set scorers using F1 score and Cohen's kappa\n",
    "scorers = {\n",
    "    'kappa': make_scorer(cohen_kappa_score),\n",
    "    'f1': make_scorer(f1_score)\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scaling & Train-Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df.pop('target')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['age', 'campaign', 'previous', 'cons.conf.idx', 'euribor3m',\n",
       "       'blue_collar', 'student', 'retiree', 'unemployed', 'illiterate',\n",
       "       'unknown_edu', 'university', 'basic4', 'basic_other', 'single',\n",
       "       'no_default', 'age_retiree', 'age_student', 'age_basic4',\n",
       "       'age_illiterate', 'poutcome_nonexistent', 'poutcome_success',\n",
       "       'contact_telephone', 'month_aug', 'month_dec', 'month_jul', 'month_jun',\n",
       "       'month_mar', 'month_may', 'month_nov', 'month_oct', 'month_sep',\n",
       "       'day_of_week_mon', 'day_of_week_thu', 'day_of_week_tue',\n",
       "       'day_of_week_wed'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, test_size=0.3, random_state=8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use **MinMax scaler** to scale the variables because (1) don't need normally distributed data given that we are not using OLS, (2) this is a sizeable dataset, and (3) the prevalence of **dummy variables** among the explanatory variables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = MinMaxScaler()\n",
    "X_train_mm = scaler.fit_transform(X_train)\n",
    "X_test_mm = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = pd.DataFrame(X_train_mm, columns=['age', 'campaign', 'previous', 'cons.conf.idx', 'euribor3m', \n",
    "                                            'blue_collar', 'student', 'retiree', 'unemployed', 'illiterate',\n",
    "                                            'unknown_edu', 'university', 'basic4','basic_other', 'single', \n",
    "                                            'no_default', 'age_retiree', 'age_student', 'age_basic4', \n",
    "                                            'age_illiterate', 'poutcome_nonexistent', 'poutcome_success', \n",
    "                                            'contact_telephone', 'month_aug', 'month_dec', 'month_jul', \n",
    "                                            'month_jun', 'month_mar', 'month_may', 'month_nov', 'month_oct', \n",
    "                                            'month_sep', 'day_of_week_mon', 'day_of_week_thu', 'day_of_week_tue', \n",
    "                                            'day_of_week_wed'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = pd.DataFrame(X_test_mm, columns=['age', 'campaign', 'previous', 'cons.conf.idx', 'euribor3m', \n",
    "                                          'blue_collar', 'student', 'retiree', 'unemployed', 'illiterate',\n",
    "                                          'unknown_edu', 'university', 'basic4','basic_other', 'single',\n",
    "                                          'no_default', 'age_retiree', 'age_student', 'age_basic4', \n",
    "                                          'age_illiterate', 'poutcome_nonexistent', 'poutcome_success', \n",
    "                                          'contact_telephone', 'month_aug', 'month_dec', 'month_jul', \n",
    "                                          'month_jun', 'month_mar', 'month_may', 'month_nov', 'month_oct', \n",
    "                                          'month_sep', 'day_of_week_mon', 'day_of_week_thu', 'day_of_week_tue', \n",
    "                                          'day_of_week_wed'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>campaign</th>\n",
       "      <th>previous</th>\n",
       "      <th>cons.conf.idx</th>\n",
       "      <th>euribor3m</th>\n",
       "      <th>blue_collar</th>\n",
       "      <th>student</th>\n",
       "      <th>retiree</th>\n",
       "      <th>unemployed</th>\n",
       "      <th>illiterate</th>\n",
       "      <th>...</th>\n",
       "      <th>month_jun</th>\n",
       "      <th>month_mar</th>\n",
       "      <th>month_may</th>\n",
       "      <th>month_nov</th>\n",
       "      <th>month_oct</th>\n",
       "      <th>month_sep</th>\n",
       "      <th>day_of_week_mon</th>\n",
       "      <th>day_of_week_thu</th>\n",
       "      <th>day_of_week_tue</th>\n",
       "      <th>day_of_week_wed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>28823.000000</td>\n",
       "      <td>28823.000000</td>\n",
       "      <td>28823.000000</td>\n",
       "      <td>28823.000000</td>\n",
       "      <td>28823.000000</td>\n",
       "      <td>28823.000000</td>\n",
       "      <td>28823.000000</td>\n",
       "      <td>28823.000000</td>\n",
       "      <td>28823.000000</td>\n",
       "      <td>28823.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>28823.000000</td>\n",
       "      <td>28823.000000</td>\n",
       "      <td>28823.00000</td>\n",
       "      <td>28823.000000</td>\n",
       "      <td>28823.000000</td>\n",
       "      <td>28823.000000</td>\n",
       "      <td>28823.000000</td>\n",
       "      <td>28823.000000</td>\n",
       "      <td>28823.000000</td>\n",
       "      <td>28823.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.283754</td>\n",
       "      <td>0.037291</td>\n",
       "      <td>0.028796</td>\n",
       "      <td>0.430109</td>\n",
       "      <td>0.677648</td>\n",
       "      <td>0.222912</td>\n",
       "      <td>0.020747</td>\n",
       "      <td>0.040731</td>\n",
       "      <td>0.024217</td>\n",
       "      <td>0.000486</td>\n",
       "      <td>...</td>\n",
       "      <td>0.129515</td>\n",
       "      <td>0.012906</td>\n",
       "      <td>0.33279</td>\n",
       "      <td>0.099886</td>\n",
       "      <td>0.017521</td>\n",
       "      <td>0.013600</td>\n",
       "      <td>0.207265</td>\n",
       "      <td>0.209728</td>\n",
       "      <td>0.196718</td>\n",
       "      <td>0.197516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.128394</td>\n",
       "      <td>0.065496</td>\n",
       "      <td>0.082555</td>\n",
       "      <td>0.193246</td>\n",
       "      <td>0.392972</td>\n",
       "      <td>0.416207</td>\n",
       "      <td>0.142540</td>\n",
       "      <td>0.197671</td>\n",
       "      <td>0.153724</td>\n",
       "      <td>0.022034</td>\n",
       "      <td>...</td>\n",
       "      <td>0.335774</td>\n",
       "      <td>0.112873</td>\n",
       "      <td>0.47122</td>\n",
       "      <td>0.299852</td>\n",
       "      <td>0.131203</td>\n",
       "      <td>0.115826</td>\n",
       "      <td>0.405354</td>\n",
       "      <td>0.407122</td>\n",
       "      <td>0.397524</td>\n",
       "      <td>0.398132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.185185</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.338912</td>\n",
       "      <td>0.160961</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.259259</td>\n",
       "      <td>0.023810</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.376569</td>\n",
       "      <td>0.957379</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.370370</td>\n",
       "      <td>0.047619</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.602510</td>\n",
       "      <td>0.980957</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 36 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                age      campaign      previous  cons.conf.idx     euribor3m  \\\n",
       "count  28823.000000  28823.000000  28823.000000   28823.000000  28823.000000   \n",
       "mean       0.283754      0.037291      0.028796       0.430109      0.677648   \n",
       "std        0.128394      0.065496      0.082555       0.193246      0.392972   \n",
       "min        0.000000      0.000000      0.000000       0.000000      0.000000   \n",
       "25%        0.185185      0.000000      0.000000       0.338912      0.160961   \n",
       "50%        0.259259      0.023810      0.000000       0.376569      0.957379   \n",
       "75%        0.370370      0.047619      0.000000       0.602510      0.980957   \n",
       "max        1.000000      1.000000      1.000000       1.000000      1.000000   \n",
       "\n",
       "        blue_collar       student       retiree    unemployed    illiterate  \\\n",
       "count  28823.000000  28823.000000  28823.000000  28823.000000  28823.000000   \n",
       "mean       0.222912      0.020747      0.040731      0.024217      0.000486   \n",
       "std        0.416207      0.142540      0.197671      0.153724      0.022034   \n",
       "min        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "75%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "max        1.000000      1.000000      1.000000      1.000000      1.000000   \n",
       "\n",
       "       ...     month_jun     month_mar    month_may     month_nov  \\\n",
       "count  ...  28823.000000  28823.000000  28823.00000  28823.000000   \n",
       "mean   ...      0.129515      0.012906      0.33279      0.099886   \n",
       "std    ...      0.335774      0.112873      0.47122      0.299852   \n",
       "min    ...      0.000000      0.000000      0.00000      0.000000   \n",
       "25%    ...      0.000000      0.000000      0.00000      0.000000   \n",
       "50%    ...      0.000000      0.000000      0.00000      0.000000   \n",
       "75%    ...      0.000000      0.000000      1.00000      0.000000   \n",
       "max    ...      1.000000      1.000000      1.00000      1.000000   \n",
       "\n",
       "          month_oct     month_sep  day_of_week_mon  day_of_week_thu  \\\n",
       "count  28823.000000  28823.000000     28823.000000     28823.000000   \n",
       "mean       0.017521      0.013600         0.207265         0.209728   \n",
       "std        0.131203      0.115826         0.405354         0.407122   \n",
       "min        0.000000      0.000000         0.000000         0.000000   \n",
       "25%        0.000000      0.000000         0.000000         0.000000   \n",
       "50%        0.000000      0.000000         0.000000         0.000000   \n",
       "75%        0.000000      0.000000         0.000000         0.000000   \n",
       "max        1.000000      1.000000         1.000000         1.000000   \n",
       "\n",
       "       day_of_week_tue  day_of_week_wed  \n",
       "count     28823.000000     28823.000000  \n",
       "mean          0.196718         0.197516  \n",
       "std           0.397524         0.398132  \n",
       "min           0.000000         0.000000  \n",
       "25%           0.000000         0.000000  \n",
       "50%           0.000000         0.000000  \n",
       "75%           0.000000         0.000000  \n",
       "max           1.000000         1.000000  \n",
       "\n",
       "[8 rows x 36 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Confirms that the max-min scaling has taken place\n",
    "X_train.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preliminary Estimations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Will evaluate 5 classification models: _Logistic regression, NaiveBayes, Random Forest, SVC & XGBoost_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dictionary of classifiers, setting class-weight='balanced' where possible, set NB prior to reflect the imbalance,\n",
    "# and scale_pos_weight in XGBoost to 8 to reflect the approximate ratio of negative class to positive class\n",
    "clf_dict = {\n",
    "    'Logistic Regression': LogisticRegression(class_weight='balanced', max_iter=3000, random_state=8),\n",
    "    'Naive Bayes': GaussianNB(priors=[0.113, 0.887]), \n",
    "    'Random Forest': RandomForestClassifier(criterion='entropy', class_weight='balanced', random_state=8),\n",
    "    'SV Classification': SVC(class_weight='balanced', random_state=8), \n",
    "    'XG Boost' : xgb.XGBClassifier(scale_pos_weight=7.85, use_label_encoder=False, seed=8)    \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch_clf(X_train, y_train, X_test, y_test, clf_dict, verbose=True):\n",
    "    '''\n",
    "    Fits a dictionary of algorithms, tests them and returns metrics\n",
    "    '''\n",
    "    train_f1_scores = []\n",
    "    test_f1_scores = []\n",
    "    train_cohen_scores = []\n",
    "    test_cohen_scores = []\n",
    "    train_pre_scores = []\n",
    "    test_pre_scores = []\n",
    "    train_acc_scores = []\n",
    "    test_acc_scores = []\n",
    "    \n",
    "    # Loop through dictionary items\n",
    "    for key, clf in clf_dict.items():\n",
    "   \n",
    "        # Fit classifier\n",
    "        clf_fitted = clf.fit(X_train, y_train)\n",
    "        \n",
    "        # Get predictions\n",
    "        train_preds = clf_fitted.predict(X_train)\n",
    "        test_preds = clf_fitted.predict(X_test)\n",
    "\n",
    "        #Get F1 scores\n",
    "        train_f1 = f1_score(y_train, train_preds, average='binary')\n",
    "        train_f1_scores.append(round(train_f1, 4))\n",
    "        test_f1 = f1_score(y_test, test_preds, average='binary')\n",
    "        test_f1_scores.append(round(test_f1, 4))\n",
    "        \n",
    "        # Get Cohen's kappa\n",
    "        train_cohen = cohen_kappa_score(y_train, train_preds)\n",
    "        train_cohen_scores.append(round(train_cohen, 4))\n",
    "        test_cohen = cohen_kappa_score(y_test, test_preds)\n",
    "        test_cohen_scores.append(round(test_cohen, 4))\n",
    "        \n",
    "        # Get AUC of precision-recall curves\n",
    "        train_pre = average_precision_score(y_train, train_preds, average='macro')\n",
    "        train_pre_scores.append(round(train_pre, 4))\n",
    "        test_pre = average_precision_score(y_test, test_preds, average='macro')\n",
    "        test_pre_scores.append(round(test_pre, 4))        \n",
    "                \n",
    "        #Get accuracy scores\n",
    "        train_acc = accuracy_score(y_train, train_preds)\n",
    "        train_acc_scores.append(round(train_acc,4))\n",
    "        test_acc = accuracy_score(y_test, test_preds)\n",
    "        test_acc_scores.append(round(test_acc,4))\n",
    "        \n",
    "    # Create results dataframe\n",
    "    results = pd.DataFrame({'Model': list(clf_dict.keys()), \n",
    "                            'Train F1': train_f1_scores,\n",
    "                            'Test F1': test_f1_scores,\n",
    "                            \"Train Cohen's kappa\" : train_cohen_scores,\n",
    "                            \"Test Cohen's kappa\" : test_cohen_scores,\n",
    "                            'Train PR-AUC': train_pre_scores,\n",
    "                            'Test PR-AUC': test_pre_scores,\n",
    "                            'Train Accuracy': train_acc_scores,\n",
    "                            'Test Accuracy': test_acc_scores,\n",
    "                            })\n",
    "\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[16:23:06] WARNING: ..\\src\\learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Train F1</th>\n",
       "      <th>Test F1</th>\n",
       "      <th>Train Cohen's kappa</th>\n",
       "      <th>Test Cohen's kappa</th>\n",
       "      <th>Train PR-AUC</th>\n",
       "      <th>Test PR-AUC</th>\n",
       "      <th>Train Accuracy</th>\n",
       "      <th>Test Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>0.4339</td>\n",
       "      <td>0.4356</td>\n",
       "      <td>0.3346</td>\n",
       "      <td>0.3361</td>\n",
       "      <td>0.2507</td>\n",
       "      <td>0.2523</td>\n",
       "      <td>0.8111</td>\n",
       "      <td>0.8103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Naive Bayes</td>\n",
       "      <td>0.4074</td>\n",
       "      <td>0.3971</td>\n",
       "      <td>0.3245</td>\n",
       "      <td>0.3104</td>\n",
       "      <td>0.2296</td>\n",
       "      <td>0.2219</td>\n",
       "      <td>0.8535</td>\n",
       "      <td>0.8464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Random Forest</td>\n",
       "      <td>0.9216</td>\n",
       "      <td>0.3647</td>\n",
       "      <td>0.9109</td>\n",
       "      <td>0.2996</td>\n",
       "      <td>0.8545</td>\n",
       "      <td>0.2149</td>\n",
       "      <td>0.9811</td>\n",
       "      <td>0.8782</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>SV Classification</td>\n",
       "      <td>0.4763</td>\n",
       "      <td>0.4635</td>\n",
       "      <td>0.3899</td>\n",
       "      <td>0.3748</td>\n",
       "      <td>0.2833</td>\n",
       "      <td>0.2725</td>\n",
       "      <td>0.8412</td>\n",
       "      <td>0.8368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>XG Boost</td>\n",
       "      <td>0.5767</td>\n",
       "      <td>0.4490</td>\n",
       "      <td>0.5075</td>\n",
       "      <td>0.3593</td>\n",
       "      <td>0.3805</td>\n",
       "      <td>0.2601</td>\n",
       "      <td>0.8734</td>\n",
       "      <td>0.8361</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Model  Train F1  Test F1  Train Cohen's kappa  \\\n",
       "0  Logistic Regression    0.4339   0.4356               0.3346   \n",
       "1          Naive Bayes    0.4074   0.3971               0.3245   \n",
       "2        Random Forest    0.9216   0.3647               0.9109   \n",
       "3    SV Classification    0.4763   0.4635               0.3899   \n",
       "4             XG Boost    0.5767   0.4490               0.5075   \n",
       "\n",
       "   Test Cohen's kappa  Train PR-AUC  Test PR-AUC  Train Accuracy  \\\n",
       "0              0.3361        0.2507       0.2523          0.8111   \n",
       "1              0.3104        0.2296       0.2219          0.8535   \n",
       "2              0.2996        0.8545       0.2149          0.9811   \n",
       "3              0.3748        0.2833       0.2725          0.8412   \n",
       "4              0.3593        0.3805       0.2601          0.8734   \n",
       "\n",
       "   Test Accuracy  \n",
       "0         0.8103  \n",
       "1         0.8464  \n",
       "2         0.8782  \n",
       "3         0.8368  \n",
       "4         0.8361  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = batch_clf(X_train, y_train, X_test, y_test, clf_dict)\n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This simple cross-model evaluation finds **SVC as the top performer** across the **test F1, Cohen's kappa and Precision-Recall AUC scores**, even though it did not have the highest test accuracy score. **XGBoost** was the second best in the test F1, Cohen's kappa and PR AUC scores, while **Logistic regression** had the third highest scores across these metrics. **Random Forest** had the highest **accuracy scores** on the train and test sets, but did poorly in the other metrics, indicating that it was focused on predicting the **majority class** of 0."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gridsearch Hyper-Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\camb7\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:297: FutureWarning: Setting a random_state has no effect since shuffle is False. This will raise an error in 0.24. You should leave random_state to its default (None), or set shuffle=True.\n",
      "  FutureWarning\n"
     ]
    }
   ],
   "source": [
    "# Use stratified KFold for the cross-validation given the imbalanced data\n",
    "kf = StratifiedKFold(n_splits=3, random_state=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "logit = LogisticRegression(penalty='l2', C=1, solver='liblinear', class_weight='balanced', max_iter=3000, \n",
    "                           random_state=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "logit_gs_params = {'penalty': [None, 'l1', 'l2', 'elasticnet'],\n",
    "                   'solver': ['lbfgs', 'liblinear', 'saga'],\n",
    "                   'C': [0.001, 0.01, 0.1, 1, 10],\n",
    "                   'class_weight': [None, 'balanced']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 120 candidates, totalling 360 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  17 tasks      | elapsed:    4.1s\n",
      "[Parallel(n_jobs=-1)]: Done 138 tasks      | elapsed:    6.4s\n",
      "[Parallel(n_jobs=-1)]: Done 360 out of 360 | elapsed:   24.5s finished\n"
     ]
    }
   ],
   "source": [
    "gs_logit = GridSearchCV(logit, logit_gs_params, scoring=scorers, refit='f1', cv=kf, verbose=2, n_jobs=-1)\n",
    "gs_logit = gs_logit.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4334378709571261"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_logit.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C': 0.1, 'class_weight': 'balanced', 'penalty': 'l1', 'solver': 'liblinear'}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_logit.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=0.1, class_weight='balanced', max_iter=3000, penalty='l1',\n",
       "                   random_state=8, solver='liblinear')"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_logit.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "gnb = GaussianNB(var_smoothing=1e-9, priors=[0.11, 0.89])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "gnb_gs_params = {'priors': [None, [0.113, 0.887]],\n",
    "                 'var_smoothing': [1e-3, 1e-4, 1e-5, 1e-7, 1e-9]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  23 out of  30 | elapsed:    0.4s remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done  30 out of  30 | elapsed:    0.4s finished\n"
     ]
    }
   ],
   "source": [
    "gs_gnb = GridSearchCV(gnb, gnb_gs_params, scoring=scorers, refit='f1', cv=kf, verbose=2, n_jobs=-1)\n",
    "gs_gnb = gs_gnb.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.408307843844616"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_gnb.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'priors': None, 'var_smoothing': 0.0001}"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_gnb.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GaussianNB(var_smoothing=0.0001)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_gnb.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Entropy criterion could help imbalanced datasets as it computes the logarithm of the probabilities of each class\n",
    "rf = RandomForestClassifier(n_estimators=100, criterion='entropy', max_features='log2', min_samples_split=20, \n",
    "                            class_weight='balanced', random_state=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_gs_params = {'n_estimators': [100, 125, 150, 175, 200],\n",
    "                'criterion': ['gini', 'entropy'],\n",
    "                'max_features': ['log2', 'sqrt'],\n",
    "                'min_samples_split': [20, 25, 30, 35, 40],\n",
    "                'class_weight': [None, 'balanced', 'balanced_subsample']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 300 candidates, totalling 900 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  17 tasks      | elapsed:   11.6s\n",
      "[Parallel(n_jobs=-1)]: Done 138 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done 341 tasks      | elapsed:  3.3min\n",
      "[Parallel(n_jobs=-1)]: Done 624 tasks      | elapsed:  6.1min\n",
      "[Parallel(n_jobs=-1)]: Done 900 out of 900 | elapsed:  9.3min finished\n"
     ]
    }
   ],
   "source": [
    "gs_rf = GridSearchCV(rf, rf_gs_params, scoring=scorers, refit='f1', cv=kf, verbose=2, n_jobs=-1)\n",
    "gs_rf = gs_rf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.494108163710905"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_rf.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'class_weight': 'balanced',\n",
       " 'criterion': 'entropy',\n",
       " 'max_features': 'log2',\n",
       " 'min_samples_split': 30,\n",
       " 'n_estimators': 175}"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_rf.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(class_weight='balanced', criterion='entropy',\n",
       "                       max_features='log2', min_samples_split=30,\n",
       "                       n_estimators=175, random_state=8)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_rf.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Support Vector Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Need to fix max_iter at a finite value to reduce the time it takes for the gridsearch to complete.\n",
    "svc = SVC(C=100, kernel='rbf', gamma=0.01, class_weight='balanced', max_iter=200000, random_state=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "svc_gs_params = {'C': [1, 10, 100, 1000, 10000],\n",
    "                 'kernel': ['poly', 'rbf'],\n",
    "                 'gamma': [0.001, 0.01, 0.1, 1],\n",
    "                 'class_weight': [None, 'balanced']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 64 candidates, totalling 192 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  17 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done 138 tasks      | elapsed: 18.9min\n",
      "[Parallel(n_jobs=-1)]: Done 192 out of 192 | elapsed: 28.0min finished\n",
      "C:\\Users\\camb7\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:249: ConvergenceWarning: Solver terminated early (max_iter=200000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    }
   ],
   "source": [
    "gs_svc = GridSearchCV(svc, svc_gs_params, scoring=scorers, refit='f1', cv=kf, verbose=2, n_jobs=-1)\n",
    "gs_svc = gs_svc.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.47550073486355293"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_svc.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C': 1000, 'class_weight': 'balanced', 'gamma': 0.01, 'kernel': 'rbf'}"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_svc.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=1000, class_weight='balanced', gamma=0.01, max_iter=200000,\n",
       "    random_state=8)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_svc.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb = xgb.XGBClassifier(booster='gbtree', gamma=1, n_estimators=20, learning_rate=0.01, objective='binary:logistic', \n",
    "                        scale_pos_weight=7.85, eval_metric='logloss', seed=8)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_gs_params = {'gamma': [1, 10, 100],\n",
    "                 'n_estimators': [10, 20, 25, 30],\n",
    "                 'learning_rate': [0.01, 0.1, 0.2],\n",
    "                 'scale_pos_weight': [3, 5, 7.85],\n",
    "                 'eval_metric': ['logloss', 'error', 'aucpr']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 324 candidates, totalling 972 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  17 tasks      | elapsed:    5.1s\n",
      "[Parallel(n_jobs=-1)]: Done 138 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done 341 tasks      | elapsed:  4.5min\n",
      "[Parallel(n_jobs=-1)]: Done 624 tasks      | elapsed:  9.0min\n",
      "[Parallel(n_jobs=-1)]: Done 972 out of 972 | elapsed: 14.6min finished\n",
      "C:\\Users\\camb7\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    }
   ],
   "source": [
    "gs_xgb = GridSearchCV(xgb, xgb_gs_params, scoring=scorers, refit='f1', cv=kf, verbose=2, n_jobs=-1)\n",
    "gs_xgb = gs_xgb.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5004594784679534"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_xgb.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'eval_metric': 'logloss',\n",
       " 'gamma': 10,\n",
       " 'learning_rate': 0.1,\n",
       " 'n_estimators': 20,\n",
       " 'scale_pos_weight': 5}"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_xgb.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "              colsample_bynode=1, colsample_bytree=1, eval_metric='logloss',\n",
       "              gamma=10, gpu_id=-1, importance_type='gain',\n",
       "              interaction_constraints='', learning_rate=0.1, max_delta_step=0,\n",
       "              max_depth=6, min_child_weight=1, missing=nan,\n",
       "              monotone_constraints='()', n_estimators=20, n_jobs=12,\n",
       "              num_parallel_tree=1, random_state=8, reg_alpha=0, reg_lambda=1,\n",
       "              scale_pos_weight=5, seed=8, subsample=1, tree_method='exact',\n",
       "              validate_parameters=1, verbosity=None)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_xgb.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The _**best_score**_ from the GridsearchCV on the training data was from **XGB**, followed by **Random Forest** and **SVC**. Let's follow up with scoring the test data using the optimized hyper-parameter settings found through the Gridsearch above."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The analysis is continued in the third file of the series"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}