Credit Card Fraud.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will using a credit card fraud data set to detect fraud action. The original data set comes from Kaggle: https://www.kaggle.com/mlg-ulb/creditcardfraud. It contains roughly 280K observations originally. I have simplified down to roughly 3000 observations. The aim of this project is to compare a few ML models and compare."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# General imports\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.pipeline import Pipeline\n",
    "%matplotlib inline\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import f1_score\n",
    "import sklearn.metrics\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.metrics import roc_curve, roc_auc_score\n",
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Time</th>\n",
       "      <th>V1</th>\n",
       "      <th>V2</th>\n",
       "      <th>V3</th>\n",
       "      <th>V4</th>\n",
       "      <th>V5</th>\n",
       "      <th>V6</th>\n",
       "      <th>V7</th>\n",
       "      <th>V8</th>\n",
       "      <th>V9</th>\n",
       "      <th>...</th>\n",
       "      <th>V21</th>\n",
       "      <th>V22</th>\n",
       "      <th>V23</th>\n",
       "      <th>V24</th>\n",
       "      <th>V25</th>\n",
       "      <th>V26</th>\n",
       "      <th>V27</th>\n",
       "      <th>V28</th>\n",
       "      <th>Amount</th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>149938</td>\n",
       "      <td>-0.445664</td>\n",
       "      <td>0.565437</td>\n",
       "      <td>0.733374</td>\n",
       "      <td>-0.999763</td>\n",
       "      <td>1.609578</td>\n",
       "      <td>1.187041</td>\n",
       "      <td>1.452018</td>\n",
       "      <td>-0.198439</td>\n",
       "      <td>-0.048416</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.195318</td>\n",
       "      <td>-0.107752</td>\n",
       "      <td>-0.100277</td>\n",
       "      <td>-0.329253</td>\n",
       "      <td>-0.375723</td>\n",
       "      <td>0.275200</td>\n",
       "      <td>-0.349132</td>\n",
       "      <td>-0.338845</td>\n",
       "      <td>50.19</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>148077</td>\n",
       "      <td>0.111788</td>\n",
       "      <td>1.124121</td>\n",
       "      <td>-0.191350</td>\n",
       "      <td>-0.443245</td>\n",
       "      <td>0.777409</td>\n",
       "      <td>-1.071823</td>\n",
       "      <td>1.068130</td>\n",
       "      <td>-0.203815</td>\n",
       "      <td>-0.246504</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.297632</td>\n",
       "      <td>-0.660398</td>\n",
       "      <td>0.144101</td>\n",
       "      <td>1.072859</td>\n",
       "      <td>-0.444134</td>\n",
       "      <td>0.079985</td>\n",
       "      <td>0.229395</td>\n",
       "      <td>0.091531</td>\n",
       "      <td>5.37</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>121831</td>\n",
       "      <td>2.087566</td>\n",
       "      <td>-1.134330</td>\n",
       "      <td>-0.593377</td>\n",
       "      <td>-1.058366</td>\n",
       "      <td>-1.104703</td>\n",
       "      <td>-0.413032</td>\n",
       "      <td>-1.112383</td>\n",
       "      <td>0.110687</td>\n",
       "      <td>-0.074055</td>\n",
       "      <td>...</td>\n",
       "      <td>0.053867</td>\n",
       "      <td>-0.082683</td>\n",
       "      <td>0.328018</td>\n",
       "      <td>-0.543838</td>\n",
       "      <td>-0.593097</td>\n",
       "      <td>-0.466010</td>\n",
       "      <td>-0.010324</td>\n",
       "      <td>-0.052221</td>\n",
       "      <td>40.00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>75683</td>\n",
       "      <td>-0.488630</td>\n",
       "      <td>1.039124</td>\n",
       "      <td>1.413263</td>\n",
       "      <td>0.110440</td>\n",
       "      <td>0.205570</td>\n",
       "      <td>-0.721798</td>\n",
       "      <td>0.677329</td>\n",
       "      <td>-0.007125</td>\n",
       "      <td>-0.568263</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.189721</td>\n",
       "      <td>-0.439538</td>\n",
       "      <td>0.084214</td>\n",
       "      <td>0.374691</td>\n",
       "      <td>-0.235030</td>\n",
       "      <td>0.077754</td>\n",
       "      <td>0.262267</td>\n",
       "      <td>0.102935</td>\n",
       "      <td>12.99</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>46080</td>\n",
       "      <td>-1.018049</td>\n",
       "      <td>0.727356</td>\n",
       "      <td>1.723606</td>\n",
       "      <td>-1.409580</td>\n",
       "      <td>0.255078</td>\n",
       "      <td>-0.648925</td>\n",
       "      <td>0.807408</td>\n",
       "      <td>0.190881</td>\n",
       "      <td>-0.605563</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.425400</td>\n",
       "      <td>-1.383973</td>\n",
       "      <td>0.085528</td>\n",
       "      <td>0.175845</td>\n",
       "      <td>-0.104718</td>\n",
       "      <td>0.462432</td>\n",
       "      <td>0.159748</td>\n",
       "      <td>0.099125</td>\n",
       "      <td>19.99</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Time        V1        V2        V3        V4        V5        V6  \\\n",
       "0  149938 -0.445664  0.565437  0.733374 -0.999763  1.609578  1.187041   \n",
       "1  148077  0.111788  1.124121 -0.191350 -0.443245  0.777409 -1.071823   \n",
       "2  121831  2.087566 -1.134330 -0.593377 -1.058366 -1.104703 -0.413032   \n",
       "3   75683 -0.488630  1.039124  1.413263  0.110440  0.205570 -0.721798   \n",
       "4   46080 -1.018049  0.727356  1.723606 -1.409580  0.255078 -0.648925   \n",
       "\n",
       "         V7        V8        V9  ...       V21       V22       V23       V24  \\\n",
       "0  1.452018 -0.198439 -0.048416  ... -0.195318 -0.107752 -0.100277 -0.329253   \n",
       "1  1.068130 -0.203815 -0.246504  ... -0.297632 -0.660398  0.144101  1.072859   \n",
       "2 -1.112383  0.110687 -0.074055  ...  0.053867 -0.082683  0.328018 -0.543838   \n",
       "3  0.677329 -0.007125 -0.568263  ... -0.189721 -0.439538  0.084214  0.374691   \n",
       "4  0.807408  0.190881 -0.605563  ... -0.425400 -1.383973  0.085528  0.175845   \n",
       "\n",
       "        V25       V26       V27       V28  Amount  Class  \n",
       "0 -0.375723  0.275200 -0.349132 -0.338845   50.19      0  \n",
       "1 -0.444134  0.079985  0.229395  0.091531    5.37      0  \n",
       "2 -0.593097 -0.466010 -0.010324 -0.052221   40.00      0  \n",
       "3 -0.235030  0.077754  0.262267  0.102935   12.99      0  \n",
       "4 -0.104718  0.462432  0.159748  0.099125   19.99      0  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the data set\n",
    "fraud = pd.read_csv('fraud.csv')\n",
    "fraud.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The data contains 31 columns. The first 30 columns are predictors and the last column (Class) is the response. There are no variable names to these columns, since they are PCA transformed. We will get rid of the Time column, and look at the distribution of the Class variable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2500\n",
       "1     492\n",
       "Name: Class, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove the Time column\n",
    "del fraud['Time']\n",
    "# Look at distribution of Class variable\n",
    "fraud['Class'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The two classes are quite unbalanced, which means we need to be careful with the evaluation metrics to use. \n",
    "\n",
    "For this project, we will fit the following 5 classification models:\n",
    "- Logistic Regression\n",
    "- Decision Tree\n",
    "- Bagged Trees\n",
    "- Random Forest\n",
    "- Boosted Trees\n",
    "\n",
    "The goal is to compare these models, and see which gives the best performance. To be fair, we will perform the evaluation using a same holdout set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separate predictors and response\n",
    "X = fraud.drop('Class', axis = 1)\n",
    "y = fraud.Class\n",
    "\n",
    "# Split your data into training and holdout set. \n",
    "# Use random state 1 and test size of 30%.\n",
    "X_train, X_holdout, y_train, y_holdout = train_test_split(X, y, test_size = 0.3, random_state = 1)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, let's train each of the 5 models using the training set. \n",
    "\n",
    "Since the response class is quite imbalanced, accuracy will be a bad metric to use here.We will use the F1-score when selecting the best hyperparameter for all models (https://en.wikipedia.org/wiki/F1_score). "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "CV=5 #cv\n",
    "score='f1' #scoring\n",
    "rs=1 #random_state\n",
    "MSL=5 #Minimum sample leaves- minimum number of observations at each terminal node"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\dira9\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:1505: UserWarning: Setting penalty='none' will ignore the C and l1_ratio parameters\n",
      "  \"Setting penalty='none' will ignore the C and l1_ratio \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score=nan,\n",
       "             estimator=Pipeline(memory=None,\n",
       "                                steps=[('scale',\n",
       "                                        StandardScaler(copy=True,\n",
       "                                                       with_mean=True,\n",
       "                                                       with_std=True)),\n",
       "                                       ('clf',\n",
       "                                        LogisticRegression(C=1.0,\n",
       "                                                           class_weight=None,\n",
       "                                                           dual=False,\n",
       "                                                           fit_intercept=True,\n",
       "                                                           intercept_scaling=1,\n",
       "                                                           l1_ratio=None,\n",
       "                                                           max_iter=10000,\n",
       "                                                           multi_class='auto',\n",
       "                                                           n_jobs=None,\n",
       "                                                           penalty='none',\n",
       "                                                           random_state=1,\n",
       "                                                           solver='newton-cg',\n",
       "                                                           tol=0.0001,\n",
       "                                                           verbose=0,\n",
       "                                                           warm_start=False))],\n",
       "                                verbose=False),\n",
       "             iid='deprecated', n_jobs=-1,\n",
       "             param_grid={'clf__C': array([1.e-10, 1.e-09, 1.e-08, 1.e-07, 1.e-06, 1.e-05, 1.e-04, 1.e-03,\n",
       "       1.e-02, 1.e-01, 1.e+00, 1.e+01, 1.e+02, 1.e+03, 1.e+04, 1.e+05,\n",
       "       1.e+06, 1.e+07, 1.e+08, 1.e+09, 1.e+10])},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring='f1', verbose=0)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train and tune Logistic Regression\n",
    "logistic_regression=Pipeline([('scale',StandardScaler()),('clf',LogisticRegression(solver = 'newton-cg', penalty = 'none',max_iter=10000,random_state = rs))])\n",
    "paras_1={'clf__C':np.logspace(-10, 10, 21)}\n",
    "LR = GridSearchCV(estimator = logistic_regression, param_grid = paras_1, cv = CV, scoring = score, n_jobs = -1) \n",
    "LR.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score=nan,\n",
       "             estimator=DecisionTreeClassifier(ccp_alpha=0.0, class_weight=None,\n",
       "                                              criterion='gini', max_depth=None,\n",
       "                                              max_features=None,\n",
       "                                              max_leaf_nodes=None,\n",
       "                                              min_impurity_decrease=0.0,\n",
       "                                              min_impurity_split=None,\n",
       "                                              min_samples_leaf=5,\n",
       "                                              min_samples_split=2,\n",
       "                                              min_weight_fraction_leaf=0.0,\n",
       "                                              presort='deprecated',\n",
       "                                              random_state=1, splitter='best'),\n",
       "             iid='deprecated', n_jobs=-1,\n",
       "             param_grid={'max_depth': range(1, 50),\n",
       "                         'max_leaf_nodes': range(2, 50)},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring='f1', verbose=0)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train and tune  Decision Tree\n",
    "decision_tree =  DecisionTreeClassifier(random_state=rs,min_samples_leaf = MSL)\n",
    "paras_2 = {'max_leaf_nodes': range(2,50),'max_depth': range(1,50)}\n",
    "DT = GridSearchCV(estimator = decision_tree,param_grid =  paras_2, n_jobs = -1, cv = CV,scoring=score)\n",
    "DT.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score=nan,\n",
       "             estimator=RandomForestClassifier(bootstrap=True, ccp_alpha=0.0,\n",
       "                                              class_weight=None,\n",
       "                                              criterion='gini', max_depth=None,\n",
       "                                              max_features=None,\n",
       "                                              max_leaf_nodes=None,\n",
       "                                              max_samples=None,\n",
       "                                              min_impurity_decrease=0.0,\n",
       "                                              min_impurity_split=None,\n",
       "                                              min_samples_leaf=5,\n",
       "                                              min_samples_split=2,\n",
       "                                              min_weight_fraction_leaf=0.0,\n",
       "                                              n_estimators=100, n_jobs=None,\n",
       "                                              oob_score=False, random_state=1,\n",
       "                                              verbose=0, warm_start=False),\n",
       "             iid='deprecated', n_jobs=-1,\n",
       "             param_grid={'n_estimators': array([ 100,  200,  300,  400,  500,  600,  700,  800,  900, 1000])},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring='f1', verbose=0)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train and tune Bagged Tree  \n",
    "Bagged_tree=RandomForestClassifier( max_features=None,random_state = rs,min_samples_leaf = MSL)\n",
    "paras_3= {'n_estimators': np.linspace(100,1000,10, dtype = int)}\n",
    "BaT = GridSearchCV(estimator = Bagged_tree,param_grid = paras_3,cv =CV, n_jobs = -1, scoring = score) \n",
    "BaT.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score=nan,\n",
       "             estimator=RandomForestClassifier(bootstrap=True, ccp_alpha=0.0,\n",
       "                                              class_weight=None,\n",
       "                                              criterion='gini', max_depth=None,\n",
       "                                              max_features='sqrt',\n",
       "                                              max_leaf_nodes=None,\n",
       "                                              max_samples=None,\n",
       "                                              min_impurity_decrease=0.0,\n",
       "                                              min_impurity_split=None,\n",
       "                                              min_samples_leaf=5,\n",
       "                                              min_samples_split=2,\n",
       "                                              min_weight_fraction_leaf=0.0,\n",
       "                                              n_estimators=100, n_jobs=None,\n",
       "                                              oob_score=False, random_state=1,\n",
       "                                              verbose=0, warm_start=False),\n",
       "             iid='deprecated', n_jobs=-1,\n",
       "             param_grid={'n_estimators': array([ 100,  200,  300,  400,  500,  600,  700,  800,  900, 1000])},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring='f1', verbose=0)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train and tune Random Forest\n",
    "Random_forest=RandomForestClassifier(max_features = 'sqrt', random_state = rs, min_samples_leaf = MSL)\n",
    "paras_4 = {'n_estimators': np.linspace(100, 1000, 10, dtype = int)}\n",
    "RF = GridSearchCV(estimator = Random_forest, param_grid =paras_4, cv = CV, n_jobs = -1, scoring = score)\n",
    "RF.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score=nan,\n",
       "             estimator=GradientBoostingClassifier(ccp_alpha=0.0,\n",
       "                                                  criterion='friedman_mse',\n",
       "                                                  init=None, learning_rate=0.01,\n",
       "                                                  loss='deviance', max_depth=3,\n",
       "                                                  max_features=None,\n",
       "                                                  max_leaf_nodes=None,\n",
       "                                                  min_impurity_decrease=0.0,\n",
       "                                                  min_impurity_split=None,\n",
       "                                                  min_samples_leaf=5,\n",
       "                                                  min_samples_split=2,\n",
       "                                                  min_weight_fraction_leaf=0.0,\n",
       "                                                  n_estimators=100,\n",
       "                                                  n_iter_no_change=None,\n",
       "                                                  presort='deprecated',\n",
       "                                                  random_state=1, subsample=1.0,\n",
       "                                                  tol=0.0001,\n",
       "                                                  validation_fraction=0.1,\n",
       "                                                  verbose=0, warm_start=False),\n",
       "             iid='deprecated', n_jobs=-1,\n",
       "             param_grid={'n_estimators': array([ 100,  200,  300,  400,  500,  600,  700,  800,  900, 1000])},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring='f1', verbose=0)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train and tune  Boosted Trees\n",
    "Gradient_boosting=GradientBoostingClassifier(min_samples_leaf = MSL, random_state = rs, learning_rate = 0.01)\n",
    "paras_5= {'n_estimators': np.linspace(100, 1000, 10, dtype = int)}\n",
    "BoT = GridSearchCV(estimator=Gradient_boosting,param_grid = paras_5, cv = CV, n_jobs = -1, scoring = score)\n",
    "BoT.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that you have trained all 5 models (hopefully you have named your classifiers __clearly__). It's time to evaluate the 5 models altogether. First, perform prediction on all 5 models using the holdout set, and report the F1 score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Classification model</th>\n",
       "      <th>F1 Scores</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>0.893773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Random Forest</td>\n",
       "      <td>0.891386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Boosted Trees</td>\n",
       "      <td>0.889706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Bagged Trees</td>\n",
       "      <td>0.885609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Decision Tree</td>\n",
       "      <td>0.878229</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Classification model  F1 Scores\n",
       "0  Logistic Regression   0.893773\n",
       "3        Random Forest   0.891386\n",
       "4        Boosted Trees   0.889706\n",
       "2         Bagged Trees   0.885609\n",
       "1        Decision Tree   0.878229"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Perform evaluation on holdout set\n",
    "LR_f1=f1_score(y_holdout,LR.predict(X_holdout))\n",
    "DT_f1=f1_score(y_holdout,DT.predict(X_holdout))\n",
    "BaT_f1=f1_score(y_holdout,BaT.predict(X_holdout))\n",
    "RF_f1=f1_score(y_holdout,RF.predict(X_holdout))\n",
    "BoT_f1=f1_score(y_holdout,BoT.predict(X_holdout))\n",
    "data_F1=[['Logistic Regression',LR_f1],['Decision Tree',DT_f1],['Bagged Trees',BaT_f1],['Random Forest',RF_f1],['Boosted Trees',BoT_f1]]\n",
    "F1_scores=pd.DataFrame(data_F1,columns=[\"Classification model\",\"F1 Scores\"])\n",
    "F1_scores.sort_values(by='F1 Scores',ascending= False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Logistic regression has the best performance, followed by random forest then boosted trees then bagged trees. As expected, a single decision tree doesn't perform too well."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ROC curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ROC Curves\n",
    "LR_yprob=LR.predict_proba(X_holdout)[:,1]\n",
    "LR_roc=roc_curve(y_holdout,LR_yprob)\n",
    "DT_yprob=DT.predict_proba(X_holdout)[:,1]\n",
    "DT_roc=roc_curve(y_holdout,DT_yprob)\n",
    "BaT_yprob=BaT.predict_proba(X_holdout)[:,1]\n",
    "BaT_roc=roc_curve(y_holdout,BaT_yprob)\n",
    "RF_yprob=RF.predict_proba(X_holdout)[:,1]\n",
    "RF_roc=roc_curve(y_holdout,RF_yprob)\n",
    "BoT_yprob=BoT.predict_proba(X_holdout)[:,1]\n",
    "BoT_roc=roc_curve(y_holdout,BoT_yprob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfEAAAGDCAYAAAA72Cm3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdeXxU1f3/8ddnJhtLCDsIKIuACgTCKkEFrWwuiPVXi7ti61Klav1K1eoXK7VutdYFldKKWEWhrtSvuBS0VYuyFVREEFBWkZ2QQALJzPn9MQtZJskEMgmXvJ+PRx6Ze++5d84MyofPuZ97jjnnEBEREe/x1XYHRERE5NAoiIuIiHiUgriIiIhHKYiLiIh4lIK4iIiIRymIi4iIeJSCuIiIiEcpiIt4iJmtNbN8M8szsx/MbJqZNSzVZpCZfWBmuWaWY2ZvmVm3Um0amdljZrY+fK3V4e3m5byvmdlNZrbMzPaa2UYze8XMMhP5eUWkYgriIt4zyjnXEMgCegN3Rg6YWTbwPjALaAN0BD4H/mNmncJtUoC5QHdgJNAIGATsAAaU856PAzcDNwFNga7Am8A5Ve28mSVV9RwRic00Y5uId5jZWuDnzrk54e2Hge7OuXPC2x8DXzrnbih13jvANufcFWb2c+D3wPHOubw43rMLsALIds4tKKfNv4AXnXN/DW9fFe7nqeFtB4wDbgGSgPeAPOfcbcWuMQv4t3PuUTNrAzwJDAbygD85556I4ysSqVOUiYt4lJm1A84CVoe36xPKqF+J0fzvwLDw66HAu/EE8LAzgY3lBfAqOB84GegGvASMMTMDMLMmwHBghpn5gLcIjSC0Db//LWY24jDfX+SooyAu4j1vmlkusAHYCtwT3t+U0P/Tm2OcsxmI3O9uVk6b8lS1fXkecM7tdM7lAx8DDjgtfOwnwKfOue+B/kAL59xE59wB59y3wF+Ai6qhDyJHFQVxEe853zmXDpwOnMjB4LwLCALHxDjnGGB7+PWOctqUp6rty7Mh8sKF7uPNAC4O77oEmB5+3R5oY2a7Iz/Ab4BW1dAHkaOKgriIRznn/g1MAx4Jb+8FPgUujNH8p4SK2QDmACPMrEGcbzUXaGdm/SposxeoX2y7dawul9p+GfiJmbUnNMz+Wnj/BuA751zjYj/pzrmz4+yvSJ2hIC7ibY8Bw8wsK7x9B3Bl+HGwdDNrYmb3AdnAveE2LxAKlK+Z2Ylm5jOzZmb2GzMrEyidc6uAp4GXzex0M0sxszQzu8jM7gg3WwpcYGb1zawz8LPKOu6cWwJsA/4KvOec2x0+tADYY2a3m1k9M/ObWQ8z638oX5DI0UxBXMTDnHPbgL8B/xve/gQYAVxA6D72OkKPoZ0aDsY45/YTKm5bAfwT2EMocDYH5pfzVjcBk4CngN3AGuDHhArQAP4EHAC2AM9zcGi8Mi+H+/JSsc8UAEYReoTuO0K3Af4KZMR5TZE6Q4+YiYiIeJQycREREY9SEBcREfEoBXERERGPUhAXERHxKAVxERERj/LcakLNmzd3HTp0qO1uiIiI1IjFixdvd861iHXMc0G8Q4cOLFq0qLa7ISIiUiPMbF15xzScLiIi4lEK4iIiIh6lIC4iIuJRCuIiIiIepSAuIiLiUQriIiIiHqUgLiIi4lEK4iIiIh6lIC4iIuJRCQviZjbVzLaa2bJyjpuZPWFmq83sCzPrk6i+iIiIHI0SmYlPA0ZWcPwsoEv451rgmQT2RURE5KiTsLnTnXMfmVmHCpqMBv7mnHPAZ2bW2MyOcc5tTlSfROTIsDdnP9s35NV2Nyr0r5Vb+XTNjtruhtSaIBYswlwhvmBR+HUR5gKhbVeELxiI/vYHC0kL7iUtkE96muMX90+skV7W5gIobYENxbY3hveVCeJmdi2hbJ3jjjuuRjonIonz4QsrWLfsyA+QmbXdATkCJId/4md79iWmKzHUZhC3GPtcrIbOuSnAFIB+/frFbCMiR64dm/LYt+dAdDtv136aH9uQIZecUIu9iu39r7bw0Tfb+G77Xjo2b8B9P+5Rps2BdesJ5uRU+drOOYIECbrQT8AFCDpH0AUIBIOhY8EAAYIEg0GCFNvvAqHzguHzAkUEi4pwRUW4QBGuKICL7AtvEyiCQAAXCB0jEISiABYIQDAIgWDodSAIAYcFg1ggiC8QhGAQXxAsEMSCDl/Q4QuEfwcdviDRn6TI6xr629kBAT8EDYJ+CPqMoBlBv+F8od9Bn0H4tfP7Q699Pkjy4Xw+nM9CHTbAHFgQCER/Qll3IRDAZw58YAY+c5jf8CWnYclp+NLq40utj6U2wJfWCH+9RiQ1aAzOhU5IsNoM4huBY4tttwO+r6W+iEiCHCgoYubvF+KCJf+G79CzOa07ZpR7nnOOomARRa4o9Lv4T4x9ARegMFhYpl0gWGx/+LxAMBDdjhyL7HtrywZ2JufT+Hg/+5qm8tjGt0pc89jPf+C8KV9Vy3djgD/8U7Vcr+qCFgp8AZ8R8B8MdEG/EfT7or9dki/02+/D+f24NB8k+XH+yG8/waQkgkl+LCkJkpI4EP5tSUlYchKWlIwvOTm8nYwvKQVfcmifPzkFS0rGn5KCPzkVf3IK/pRU/Ekp0X3JKWkkpaSSlJyGPyWFpJS00L7ktNB1/f7YH9I52LcT8n6A3B8gbwvkbobcLeF9xX4X5Zc9P7k+NGwF6a1DPw1bQ3qrg7/Tjwkdr9ekRgJ0PGoziP8DGGdmM4CTgRzdD5e67qX565m1dFOZ/QEKKPCVu6TwIXI4grhw5uFcgEb5yfgDFt4XBILhNqEfCIIFcOFzIRDd7yx0rUZ795ORVxBt43cpdA3+mM31F7Kz3vJwe8eCLT/w0r37cNFruOj1HMFwZlRDnGH4AB+tndHG/DTal4p/h5+81T78Pj9+89PQJXHGjA3kpSfz1hWdsaSk8LEkksJt/OaPti+zHf7tMx9JvqRSx5Pw+3z4zU+SJR1s6/PhJ9Qmst9nPvxJKSSFA2BSOPD5k1NJigbAVHzJyRAJpj4PP1EcDMDe7bBrXSgA524OB+gfDv6OvA4Wlj0/tdHB4Nyu/8HXDVsXC9itIDX9iAnO8UpYEDezl4HTgeZmthG4h/A/Np1zk4HZwNnAamAfMDZRfRHxillLN7F88x66HdOoxP6tSa+zK+lfCX3vpnvbcO7Xtyfs+j/67wqO+WFBwq5/+AKltveW27LdpCfpP3RoYrtTFwQKIW/rwcy5eFAuHpz3bgNX+s+HUEYcyY6bdykWnMNZcySLTqlf85+thiSyOv3iSo474MZEvb9ITXPOsXzncvYX7T/ka+yzVXRoA7ePblFi/93/WYMVNuWRIY/Ef60tAQIFFd+k9BXLDvO3BFnxxW5OGp5B4+BOfGb4fT4MXzR79JsPs4ozui2//z31+/WjYbEg5/MZLa68Hp/vF3H3/0jlb9KYtK5da7sbR7bCglAQLnc4O/yzbwdlS6EMGrQ4GIBb9yiVMUeGuFtBUmptfLojSm0Op4tUu/KGo2tCnm8561MeO7yLhP9OuvLdsoc6N+5M/9b9K72Ec47P3lzDsvfWH1IXWn8xD5vxdJn94cH0SjUBmnX+ES3PH3RI7y9HsAN7i2XMpe4xFx/iLthd9lzzhzPkVpBxLLTrV3Y4O701NGgJfoWmeOmbkqNKecPRcXOOhvkOO4QqW58PWu5vT9OiH5FMy0N7f6Bn2wz6tW9aZv8xDY7hh+8qr4j+et5mln/8Pd1ObUPnvhX3wxUVcWDDxuhQZXKSYW98TUHjxrR99I+H9gHMqNer16GdKzXPOdi/p5zh7FJZ9IHcsuf7Uw5mx806Q4dTS95vjgTn+s3AV05BmhwyBXHxlMoy7UgAn3ld9iFd/6uPN/Gv6SsPsXcdgVsP8dxiVsBKyg7Jr2QtsDauS/QZ2Z6BozthlRTpbJ/yF/Y++mh0O1ISlNKhAw0GKZP2tJiV2uVk0ZVVarfqAZ2HlazUjgTpI6hSuy5SEJdK5ecdIHdHQW13A4C58zawdXsenZo3jHl8QEZDhrRrxtZ1e0rs/z7vewoClX+GH74Ltel+SQa+pCr8xZSXx7r1y/hw/Yf8LPNntKh/6Jn44UpL89GiVQEFXy2vtG3unDmkdulMy/HjS+xP6dAhQb2Twxap1K5oODtyPzpwoOz5FVZqFwvOHqzUrossVF/mHf369XOLFi2q7W7UKS/dO59dm8uv1D3aBKyIqQNuJ+Arivucp58qovmeytsdiZpdfx0tb7mltrshJSq1t5RfsZ23tfxK7dLPMxcfzo68TmlQ859NDouZLXbO9Yt1TJl4HZWzLZ/C/fEFqYK8Axx7UhMyzzi28sYJ8vC7K9iwcx/HNq3PyR2bMeSEFjHb7cjfQUGpocG1e9YyY8UMRh1/Hm3qtcK3bWc5cwOGJNcLMqnp/1Spf00P/IGC7BOxUcM4vvHxVTq3NpnfR/3+lRfLyWEo2l/5cHbeD6HsOmaldvOD2XHxSu0Sj1GpUruuUhCvg3ZsymPG76r2vG7j1g3o2LN5gnp0UHn3vJfv20u3jo14poJ73ev3rGfsGz+JfbApnDrof2k5819sf3JSdXW3hLY9s2l5wc8Tcm05AkUqtSuaFSx3cwWV2i1DwTijHbTrW3Y4O7116FErf6LnchMvUxA/iu3ZkU/R/rIPBW3bEKow7X9uR5q3jX1vuQSDNl0ax/We2/O3k7M/dgW1CwZh/aZQwU05/vXBCnJ27uO4piUnZ8hOhZMzmrF6Sfn3tdftWUfb7Y4Lu/yEzo07lziWlpTG8btS2Ll+PZaWRps/PBzX54mX+XzUHzCgWq8ptSBaqV3BcHYkYJdbqR0evm52PHQ4JfYQtyq1pZooiB+ltq3P5e/3L6ywzXHdmtK6U/lzV1fVvsJ9jHh1BAeCMYppgHPnB7nig4qfNP5VJe8RY0LFqDbAnwCYGfP4d+Hf/ubNaTRsWCXvJEcV5yB/VzgolzdlZwWV2kn1DmbHrXpA56FlZwVTpbbUAgXxo4xzjtwdBWzfGFqrecCojjRuVXbKweRUP606VP1Z6qJgERtzN0bfy23eAkWhe+u5B3Jptm0/w9oP44vVTdmSm0+r9HrRcztvX0rQvuaFcyoOoMe3aMgJx8QxQhBDqi+Vrk264q8gy1Hl9VEkGIR92yueFSzeSu22/WJP2ZneKtROwVmOQAriR5lVC7fwz6kHHy1q36MZLdsf4sQnMTy88GFeXvEyAAO/DnLrmwcz6xTgcQDe5dxyzvfVr8+Djzxebf2Ro1TMSu1Yj1HFUandrHP5Fduq1BaPUxD3sLxd+wkUlRye3vVDaDH60y89gQYZqbQ4Nr3K1y0KFrF578EF5Vx+Pm5HqDgnb+0aTshvzDWZ19Bww2LgfbbdeAHvfrebnfsO0Kx+PVJdO3wk07d9EwYdX7IYLuW42qtwlyNA6Urt8iq2K63UbhUa1i49ZWfDcKV2clptfDqRGqcg7lHrlu3g/yZ9HvugQZf+rUhJq9ofb6QyfHPSdHYl/Tu6/6GpRXTcEnp9eXTvA9FXzwbP4L+NAnQ7oRGTD3GmNPG4EpXaFVRs5+8qe26kUrthK8hoG6NSO/y7YUtVaouUoiDuUTnbQhn3oP/XmXrpJf9ia9A4Ne4A7pxj676tOBzv/GceW7buoWHr70n2pdO8cCQAzXPfZHW7DOb3OAGAZJqR7JoAkNugMQVpDeh2DIzOaltdH0+OBGUqtctZxzlvS6hdab7kg9lx6Urt4hXbqtQWOWQK4h60cv4PfDxzFQCPrtjIgZQYBTf/Lrsrlh3+f7Il+RV6rw7ym1dKV46/HH113Pk/ZtSE/z3EHssRpXildkXBOfeHCiq1wwG4VXfofGaMdZxVqS1SExTEj3CBoiAFe0s+WLV7SygLf79xEWmxAnjx88knGGMxjYgDthWcn/Z7BgKfsGjUqbTp0IK2DdvQukHrUCMzGpx62mF9DqkB0UrtWMPZpTLnWJXaKekHg3PbvjGm7FSltsiRRkH8CFDRylw9V+2ncV7ZZ6sdjvrtG/JyBfegf9j7A2e9dhZFruLpVdOT0/mfwWex+b1PuOiW35LcVsPiR5RAEezdWvk6zuVVaqc1PvjIVPtTys4KpkptEc9SED8ClFgD2zmSiv09nHrAsae+saVZyT+qgmTjnN7HsGPHRjgQewqUDTnfUm9vIaOPH03nJp1jtgE4Lr09wQXlL+8pCVK0v1R19pZiQbqySm1CU3KWqNSOEZxVqS1yVFMQryXFs+/ia2B/+sYa/vveuhJte/dpzZlXdStzjbunXkLvsUvwlTOLaUPgWQBer7Q/WyIvklT9e9hiVmrHGOKOWantOxh8M9pC2z5lh7NVqS0iYQritaR49t3tmEaMzmpL4YEAOdvySWuQTP9zO0bbtu/RrMz5gby92IYf8DnYMjqbAy1jT5+a7Eume7PuJPkq/6NOat6M5Fa1tw72Ec052J8bYzg7xv3meCq12w8qOytYw9ah56BVqS0icVIQr0HlZd8AO7/fy19v+Yhg0NGkdX16ntGu3OvseecdNv3qVi4Jbw+8ajxpJ52U6O4fnWJVapcXnAv3lT0/UqndsDW06gbH/yjGBCStoX5TFYOJSLVTEK9B5WXfLujYsz2fYNCReXo7Omc1JS9ne7nX2fvtagBmj2xGg5ZtOPGEE2rqI3hHmUrtH2IE53DArqhSu2Hr8JB2jCk701urUltEapWCeAKUuyZ2qez763mbmXJTyQe6O/fMYMdlw8nJi/F8bun36bGbM7sMwny+6um4FxSv1C5vys7c8IIX5VZqhwNx+0Flh7Mjx1IPbQEWEZGapCCeACWqzYvpdkwjzuvZhkBh6JGxnK3hWdcuCFWOJ6f5yUg/QE5ePgtO8NNqQPnPZh9omcEvB57I4HaDE/Qpali0Urv0cHapIe7yKrXrNz+YHbfsXn5wVqW2iBxFFMSr2Uvz1zP/u52c3LFpNOOO2LEpj1ceWMTk59dE9/l8Ru/hxwGwdeM3rDx7NPWAr09I48q7nqnJrifGgX2VzwpWUaV2g5ahANyoLbTpE3sCElVqi0gdpSBezWYt2YgFHef1aEWgsAjnHEEXyrx3b80jUBSk26mtadgkFYCMlvUoKgzdk92xYRX1DsCqQcdy9lW31NpnqFSkUruiWcEqq9SOBOJopXaMdZxVqS0iUiEF8WrW/dsDDN5Tn90vrGXyC2tjtqn/+K9omLsegACwqtTxZsPP4pRuZye0nzGVW6kdo2I7ZqV22sHsOFqpXXod5/Cc2nXpPr6ISIIoiFezevsdKfu3cdzer0k6oQuLtywmI7UxDVNChVI+K2T7yLbssDYxz/elpnHqWZfHPHbIgkHYt6PUcHbpIe5wMVggxjzrpSu1Y03Z2bAVpGWoUltEpAYpiB+CLWv38OYf/0tRYdk5zesDTXZ9R07ha/yuM9AZ7hxwJ5ecdEmZtoctVqV2rHWc926FYIz500tUameXHc5WpbaIyBFNQfwQ7NmeT1FhkB6D2/L2N1vYlrufFumpnLLwHfLTGuA67GdGS8c5nc6lY6OODO8wvGpvEKtSO2Zw3kaFldoNW0HLbmWn7EyPzKldr1q+DxERqR0K4lUwbcZX7Pn3ZnwODB9pf7yOy/Ojs46TFIR3+vv5W58kAs74defzGXjMwPjfYOsKePcO+PbDsseildqtIL1N2UrtSHBu0BKSUqrh04qIyJFOQbwKvlqxgw7Ox5bU9zlleR6fH5/K3nqdadogheYNU8Eg/bQTGNumMfWT6pPVIiu+CxfkwL8eggV/hpSGcNr/QOP2JSu2VaktIiKlKIhX4pGFj7B36guM/vgApx17Nhvan8NP/jmL5CCc/ue3Se3U6dAvHgzCFzPhnxNCQ+N9r4QfTYAGZRc8ERERKU1BvBIrdq7g9G0+LDWVtcc0wQ/sGJVN5vHZpHTocOgX/n4pzB4PGxdA235wycxQ5beIiEicFMTLcfMHN9PtL//iF8sPkBRIZdGAu3EpTQA4/cFnsUN9lGrfTpg7ERZPCw2Rj34ael2s56ZFRKTKFMTL8eX2Lxm92XAZ6eQPOoOC3U3ZmBSgW//WhxbAg4FQ4P7gd1CwB06+Hk6/A+o1rva+i4hI3aAgXowrLOS7MWPI37ieBw/spcF+WNj7SnJye5EE7G+TxtgrM6t+4fXzYfZt8MMX0OE0OOvh0IxmIiIih0FBvJhgfj77l3/N3pPa8Un9fbC3B8FmJ+BzsLllEgMGta3aBXO3wJx74POXQ4+F/WQqdL9As5qJiEi1UBCPYdfAE3iu9Q/0yrmO7OVG604Z3PrrvvFfIFAI8/8M/3oQigrg1FtDj41p5jMREalGCuIVqB+eRrzdiU3iP+nbf8HsX8P2ldB5GJz1UGilLhERkWqmIB7DB+s/5Ee5V9BhV2jYu32POJ7b3r0B3r8Lls+CJh3g4hnQdaSGzkVEJGEUxMvRNucE0hokkzmgNc2PrWAYvLAA5j0JH/8xtH3G3TDol5CcVjMdFRGROktBPKxwf4Dn7/oYGziRNgHwB9M5sWcLTv1Jl/JPWvkuvHs77FoLJ50HI34PjY+rsT6LiEjdpiAetjdnP/sL65FRsIlvWu6kXWsfJw3qF7vxjjWhhUpWvQ/Nu8Llb8LxZ9Rsh0VEpM6r80H8pfnrmb1wI93W7CSD+jTe+R9mn/Vfrsm8htadMko2PrA3NGw+70nwp8Lw+2DAdVo1TEREakWdD+Kzlm5ix/d5ZOfXxxUup3Het4zqNIoRHUYcbOQcfPU6vP+/sGcT9LwIht0bWmVMRESkltTpIP7S/PXM/24nQ1tmwM4D1A/Mp5XfuP+0+w822rIc3vk1rP0YWmeGJmw5rgprhIuIiCRInQ7is5ZuAmBwlxbkfrOp5MGCnNBkLfP/DKnpcM4foe9YrektIiJHjDodxBvkB7lufwp7Zi/DaEKHtftCB5ZMD02Xune71vgWEZEjVp0O4g33BWmU7yd17/ek568itV2Qpg33w6wboF1/uPQVaNO7trspIiISU0KDuJmNBB4H/MBfnXMPljp+HPA80Djc5g7n3OxE9imWHstfptGxaXTr/N/QGt9Dtca3iIgc+RIWxM3MDzwFDAM2AgvN7B/OueXFmt0N/N0594yZdQNmAx0S1afy5KdCo5McND4Wrv8E0jIqP0lERKSWJTLVHACsds5965w7AMwARpdq44BG4dcZwPcJ7E8Z6bm7AFhzjEH3hlCviQK4iIh4RiKDeFtgQ7HtjeF9xf0WuMzMNhLKwn+ZwP6U0WTPNgBWtTE6Bv1gqjwXERHvSGQQj7V8lyu1fTEwzTnXDjgbeMHMyvTJzK41s0Vmtmjbtm3V3tEG2dnUw+nxMRER8ZREBvGNwLHFtttRdrj8Z8DfAZxznwJpQPPSF3LOTXHO9XPO9WvRokViehsMKBMXERFPSWQQXwh0MbOOZpYCXAT8o1Sb9cCZAGZ2EqEgXv2pdjxcUJm4iIh4SsKCuHOuCBgHvAd8TagK/Sszm2hm54Wb/Q9wjZl9DrwMXOWcKz3kXjOCASg7ki8iInLESuhz4uFnvmeX2jeh2OvlwCmJ7EPcXAB8abXdCxERkbgp9YzQPXEREfEYBfEI3RMXERGPURCPcMrERUTEWxTEI4LKxEVExFsUxCOcqtNFRMRbFLUiggFl4iIi4ikK4hG6Jy4iIh6jIB6hTFxERDxGQTzCBZWJi4iIpyiIRwQD4NPXISIi3qGoFaF74iIi4jEK4hG6Jy4iIh6jIB6hTFxERDymTgdxx4GDG5qxTUREPKZOB/FC2w1AqwatlImLiIjn1OkgHvGjY89QdbqIiHiOolaEMnEREfEYBfEIVaeLiIjHKIhHKBMXERGPURAHcC70W5m4iIh4iII4hOZNB2XiIiLiKQriEHpGHFSdLiIinqKoBaH74aBMXEREPEVBHHRPXEREPElBHHRPXEREPElBHIDIPXEFcRER8Q4FcSiWievrEBER71DUAgjqnriIiHiPgjioOl1ERDxJQRwOBnFl4iIi4iEK4nDwETNl4iIi4iEK4gDonriIiHiPgjgUuyeur0NERLxDUQtUnS4iIp6kIA6qThcREU9SEIdiq5gpiIuIiHcoiAPRaVeViYuIiIcoiEOxR8z0dYiIiHcoagEHF0DR1yEiIt6hqAUH74lrOF1ERDwkriBuZilm1jnRnak1ToVtIiLiPZUGcTM7B/gS+Gd4O8vM3kh0x2pUUI+YiYiI98STiU8ETgZ2AzjnlgJHWVauyV5ERMR74gnihc653aX2uUR0ptbonriIiHhQUhxtvjaznwI+M+sI3Ax8lthu1TRVp4uIiPfEE7XGAX0JRbrXgQJCgfzo4ZSJi4iI98STiY9wzt0O3B7ZYWYXEAroRwdVp4uIiAfFk4nfHWPfXdXdkVqlTFxERDyo3EzczEYAI4G2ZvZosUONiN5EPko4VaeLiIj3VDScvhVYRuge+FfF9ucCdySyUzVOmbiIiHhQuUHcObcEWGJm051zBTXYp5oXVHW6iIh4TzxRq62ZzTCzL8zsm8hPPBc3s5FmttLMVptZzOzdzH5qZsvN7Csze6lKva82kVXMlImLiIh3xFOdPg24D3gEOAsYSxz3xM3MDzwFDAM2AgvN7B/OueXF2nQB7gROcc7tMrOWVf4E1cGFp13VPXEREfGQeDLx+s659wCcc2ucc3cDZ8Rx3gBgtXPuW+fcAWAGMLpUm2uAp5xzu8LX3xp/16uR09zpIiLiPfEE8f1mZsAaM7vezEYB8WTMbYENxbY3hvcV1xXoamb/MbPPzGxkrAuZ2bVmtsjMFm3bti2Ot66iyCSyysRFRMRD4gnivwIaAjcBpxDKnq+O4zyLsa/0nOtJQBfgdOBi4K9m1rjMSc5Ncc71c871a9GiRRxvXUWqThcREQ+q9J64c25++GUucDmAmbWL49obgWOLbbcDvo/R5jPnXCHwnZmtJBTUF8Zx/eoTWYpU1ekiIuIhFUYtM+tvZuebWfPwdncz+xvxLYCyEOhiZh3NLAW4CPhHqTZvEr6/Hn6PrsC3VfwM1UCZuIiIeE+5QdzMHgCmA5cC75rZXcCHwOeEgm2FnHNFhBZPeXXY/HAAACAASURBVA/4Gvi7c+4rM5toZueFm70H7DCz5eFrj3fO7TicD3RIgpqxTUREvKei4fTRQC/nXL6ZNSU0FN7LObcy3os752YDs0vtm1DstQNuDf/UIlWni4iI91Q0nF7gnMsHcM7tBFZUJYB7ilYxExERD6ooE+9kZpHlRg3oUGwb59wFCe1ZTXKasU1ERLynoiD+/0ptT0pkR2qV5k4XEREPqmgBlLk12ZFa5YLKwkVExHOUekIoiOt+uIiIeIyCOAABZeIiIuI5cQdxM0tNZEdqVdApExcREc+pNIib2QAz+xJYFd7uZWZPJrxnNUr3xEVExHviycSfAM4FdgA45z4nvqVIvSMYVGW6iIh4TjyRy+ecW1dqXyARnak1qk4XEREPqnQVM2CDmQ0AnJn5gV8C3yS2WzVN1ekiIuI98WTivyA0t/lxwBZgYHjf0UOZuIiIeFA8mXiRc+6ihPekNgWViYuIiPfEk4kvNLPZZnalmaUnvEe1wQXBVNgmIiLeUmnkcs4dD9wH9AW+NLM3zezoysydnhMXERHviSv9dM7Nc87dBPQB9gDTE9qrmqZ74iIi4kHxTPbS0MwuNbO3gAXANmBQwntWk1xAmbiIiHhOPIVty4C3gIedcx8nuD+1xCkTFxERz4kniHdyzgUT3pPapBnbRETEg8oN4mb2R+fc/wCvmZkrfdw5d0FCe1aTdE9cREQ8qKJMfGb496Sa6EitCuqeuIiIeE+5Qdw5tyD88iTnXIlAbmbjgLmJ7FjN0j1xERHxnnhuBF8dY9/PqrsjtcppxjYREfGeiu6JjwEuAjqa2evFDqUDuxPdsZqle+IiIuI9Fd0TX0BoDfF2wFPF9ucCSxLZqRoXDKg6XUREPKeie+LfAd8Bc2quO7XEoUxcREQ8p6Lh9H8754aY2S5CYS56CHDOuaYJ711NCQbAl1LbvRAREamSiobTzwj/bl4THalVWsVMREQ8qNzIVWyWtmMBv3MuAGQD1wENaqBvNUiPmImIiPfEk36+CTgzOx74G3AS8FJCe1XT9IiZiIh4UDxBPOicKwQuAB5zzv0SaJvYbtUwF9BwuoiIeE48kavIzC4ELgf+L7wvOXFdqgXKxEVExIPinbHtDEJLkX5rZh2BlxPbrRoW1GQvIiLiPZUuReqcW2ZmNwGdzexEYLVz7veJ71oNck6ZuIiIeE6lQdzMTgNeADYReka8tZld7pz7T6I7V2O0FKmIiHhQpUEc+BNwtnNuOYCZnUQoqPdLZMdqlHOadlVERDwnnsiVEgngAM65r4GjbHqzgDJxERHxnHgy8f+a2Z8JZd8Al3K0LYCie+IiIuJB8QTx64GbgF8Tuif+EfBkIjtV45wycRER8Z4Kg7iZZQLHA2845x6umS7VAmXiIiLiQeXeEzez3xCacvVS4J9mdnWN9aqmqTpdREQ8qKJM/FKgp3Nur5m1AGYDU2umWzUsGFR1uoiIeE5FkWu/c24vgHNuWyVtvU2ZuIiIeFBFmXgnM3s9/NqA44tt45y7IKE9q1GaO11ERLynoiD+/0ptT0pkR2qVqtNFRMSDyg3izrm5NdmRWqdMXEREPObovc9dVcrERUTEYxTEI1SdLiIiHhN35DKz1ER2pNYpExcREY+pNIib2QAz+xJYFd7uZWZH17SroHviIiLiOfFk4k8A5wI7AJxznwNnxHNxMxtpZivNbLWZ3VFBu5+YmTOz2lveVJm4iIh4TDxB3OecW1dqX6Cyk8zMDzwFnAV0Ay42s24x2qUTWmBlfhx9SRxl4iIi4jHxBPENZjYAcGbmN7NbgG/iOG8AsNo5961z7gAwAxgdo93vgIeBgng7nRCmwjYREfGWeCLXL4BbgeOALcDA8L7KtAU2FNveGN4XZWa9gWOdc/9X0YXM7FozW2Rmi7Zt2xbHWx8CZeIiIuIxla4n7pzbClx0CNe2WJeLHjTzAX8CroqjD1OAKQD9+vVzlTQ/NLonLiIiHlNpEDezv1As+EY4566t5NSNwLHFttsB3xfbTgd6AP8yM4DWwD/M7Dzn3KLK+lXtlImLiIjHVBrEgTnFXqcBP6bkMHl5FgJdzKwjsIlQNn9J5KBzLgdoHtk2s38Bt9VKAAdl4iIi4jnxDKfPLL5tZi8A/4zjvCIzGwe8B/iBqc65r8xsIrDIOfePQ+xzYigTFxERj4knEy+tI9A+nobOudnA7FL7JpTT9vRD6Ev1UXW6iIh4TDz3xHdx8J64D9gJlDtxi2cpExcREY+pMIhbqOKsF6F72gBB51xiqsNrm+6Ji4iIx1Q4hhwO2G845wLhn6MzgIMycRER8Zx4bgQvMLM+Ce9JbVMmLiIiHlPucLqZJTnnioBTgWvMbA2wl9AkLs45d3QFdmXiIiLiMRXdE18A9AHOr6G+1C5Vp4uIiMdUFMQNwDm3pob6UruUiYuIiMdUFMRbmNmt5R10zj2agP7UHt0TFxERj6koiPuBhsReyOToo0xcREQ8pqIgvtk5N7HGelLblImLiIjHVFTNVTcy8Ahl4iIi4jEVBfEza6wXRwJVp4uIiMeUG7mccztrsiO1Tpm4iIh4jNLPCN0TFxERj1EQj1AmLiIiHqMgHqFMXEREPEZBPEKZuIiIeIyCeISq00VExGMUuSIUxEVExGMUuSI0nC4iIh6jIB6hwjYREfEYBfEIZeIiIuIxCuIRysRFRMRjFMQjfPoqRETEWxS5IpSJi4iIxyiIR+ieuIiIeIyCeIQycRER8RgF8Qhl4iIi4jEK4hHKxEVExGMUxCNUnS4iIh6jyAXKwkVExJMUxAF9DSIi4kWKXqAVzERExJMUvUD3w0VExJMUvUCZuIiIeJKiF4BZbfdARESkyhTEQdXpIiLiSQrioExcREQ8SUEcAGXiIiLiPQrioOp0ERHxJEUvADScLiIi3qMgDsrERUTEkxS9QM+Ji4iIJyl6gTJxERHxJEUvQF+DiIh4UVJtd+CIoOF0EaklhYWFbNy4kYKCgtruitSytLQ02rVrR3JyctznKIgDqk4XkdqyceNG0tPT6dChA6aJp+os5xw7duxg48aNdOzYMe7zlIIC+DTZi4jUjoKCApo1a6YAXseZGc2aNavyiIyCOGg4XURqlQK4wKH9d6DoBeDT/0AiUnc1bNjwsK/x/fff85Of/KTc47t37+bpp5+Ou31pV111FR07diQrK4tevXoxd+7cw+pvdZs8eTJ/+9vfavx9ExrEzWykma00s9VmdkeM47ea2XIz+8LM5ppZ+0T2p1xaxUxE5LC0adOGV199tdzjpYN4Ze1j+cMf/sDSpUt57LHHuP766w+5r8UVFRVVy3Wuv/56rrjiimq5VlUkLIibmR94CjgL6AZcbGbdSjVbAvRzzvUEXgUeTlR/KqThdBGREtatW8eZZ55Jz549OfPMM1m/fj0Aa9asYeDAgfTv358JEyZEs/i1a9fSo0cPAL766isGDBhAVlYWPXv2ZNWqVdxxxx2sWbOGrKwsxo8fX6J9IBDgtttuIzMzk549e/Lkk09W2Lfs7Gw2bdoU3V68eDFDhgyhb9++jBgxgs2bNwOwcOFCevbsSXZ2NuPHj4++37Rp07jwwgsZNWoUw4cPB0L/QOjfvz89e/bknnvuAWDv3r2cc8459OrVix49ejBz5kwA7rjjDrp160bPnj257bbbAPjtb3/LI488AsDSpUsZOHAgPXv25Mc//jG7du0C4PTTT+f2229nwIABdO3alY8//vhw/oiAxFanDwBWO+e+BTCzGcBoYHmkgXPuw2LtPwMuS2B/KqDhdBGpffe+9RXLv99Trdfs1qYR94zqXuXzxo0bxxVXXMGVV17J1KlTuemmm3jzzTe5+eabufnmm7n44ouZPHlyzHMnT57MzTffzKWXXsqBAwcIBAI8+OCDLFu2jKVLlwKhoB8xZcoUvvvuO5YsWUJSUhI7d+6ssG/vvvsu559/PhB6RO+Xv/wls2bNokWLFsycOZO77rqLqVOnMnbsWKZMmcKgQYO4446Sg8GffvopX3zxBU2bNuX9999n1apVLFiwAOcc5513Hh999BHbtm2jTZs2vP322wDk5OSwc+dO3njjDVasWIGZsXv37jL9u+KKK3jyyScZMmQIEyZM4N577+Wxxx4DQpn/ggULmD17Nvfeey9z5syJ7w+kHIlMQdsCG4ptbwzvK8/PgHcS2J/yKRMXESnh008/5ZJLLgHg8ssv55NPPonuv/DCCwGix0vLzs7m/vvv56GHHmLdunXUq1evwveaM2cO119/PUlJobyyadOmMduNHz+eTp06cdlll/Gb3/wGgJUrV7Js2TKGDRtGVlYW9913Hxs3bmT37t3k5uYyaNCgmH0dNmxY9H3ef/993n//fXr37k2fPn1YsWIFq1atIjMzkzlz5nD77bfz8ccfk5GRQaNGjUhLS+PnP/85r7/+OvXr1y9x3ZycHHbv3s2QIUMAuPLKK/noo4+ixy+44AIA+vbtW+IfMocqkZl4rPTWxWxodhnQDxhSzvFrgWsBjjvuuOrqX7E30D1xEal9h5Ix15SqVE5fcsklnHzyybz99tuMGDGCv/71r3Tq1Knc9s65uK7/hz/8gQsuuIAnnniCK6+8ksWLF+Oco3v37nz66acl2kaGsMvToEGDEu9/5513ct1115Vpt3jxYmbPns2dd97J8OHDmTBhAgsWLGDu3LnMmDGDSZMm8cEHH1Ta94jU1FQA/H5/tdyPT2QKuhE4tth2O+D70o3MbChwF3Cec25/rAs556Y45/o55/q1aNGi+nuqudNFREoYNGgQM2bMAGD69OmceuqpAAwcOJDXXnsNIHq8tG+//ZZOnTpx0003cd555/HFF1+Qnp5Obm5uzPbDhw9n8uTJ0aBW0XC6z+fj5ptvJhgM8t5773HCCSewbdu2aBAvLCzkq6++okmTJqSnp/PZZ59V2FeAESNGMHXqVPLy8gDYtGkTW7du5fvvv6d+/fpcdtll3Hbbbfz3v/8lLy+PnJwczj77bB577LHo7YGIjIwMmjRpEr3f/cILL0Sz8kRIZCa+EOhiZh2BTcBFQInxDDPrDfwZGOmc25rAvlRMz2iKSB22b98+2rVrF92+9dZbeeKJJ7j66qv5wx/+QIsWLXjuuecAeOyxx7jsssv44x//yDnnnENGRkaZ682cOZMXX3yR5ORkWrduzYQJE2jatCmnnHIKPXr04KyzzuLGG2+Mtv/5z3/ON998Q8+ePUlOTuaaa65h3Lhx5fbXzLj77rt5+OGHGTFiBK+++io33XQTOTk5FBUVccstt9C9e3eeffZZrrnmGho0aMDpp58es68Q+kfE119/TXZ2NhB65O7FF19k9erVjB8/Hp/PR3JyMs888wy5ubmMHj2agoICnHP86U9/KnO9559/nuuvv559+/bRqVOn6HeXCOZczBHu6rm42dnAY4AfmOqc+72ZTQQWOef+YWZzgExgc/iU9c658yq6Zr9+/dyiRYuqpX/3j3uAjKKTGd5tGl1uqvnn+0REvv76a0466aTa7kbc9u3bR7169TAzZsyYwcsvv8ysWbNqu1sx5eXlRavnH3zwQTZv3szjjz9ey72qWKz/HsxssXOuX6z2CZ073Tk3G5hdat+EYq+HJvL946bCNhGRuCxevJhx48bhnKNx48ZMnTq1trtUrrfffpsHHniAoqIi2rdvz7Rp02q7S9VOC6CA7omLiMTptNNO4/PPP6/tbsRlzJgxjBkzpra7kVCKXoC+BhER8SJFL1AmLiIinqToBbonLiIinqToBQriIiLiSYpegOZOF5G6zO/3k5WVRffu3enVqxePPvoowWDwkK41YcKECucDr44lO7/88kuysrLIysqiadOm0SVKhw49Mh54qkmqTgfwadpVEam76tWrF515bOvWrVxyySXk5ORw7733VvlaEydOrPB4dSwhmpmZGe3vVVddxbnnnhtzbfKioqLofOxHK2XioOF0EZGwli1bMmXKFCZNmoRzjkAgwPjx46PLdP75z3+Otn344YfJzMykV69e0VXCrrrqqug64bWxZOecOXMYOnQoF110Eb179wZCM6hFlka94YYboqMM77zzDtnZ2fTp04cxY8awd+/ew/z2at7R/U+UeCmIi8iR4J074Icvq/earTPhrAerdEqnTp0IBoNs3bqVWbNmkZGRwcKFC9m/fz+nnHIKw4cPZ8WKFbz55pvMnz+f+vXrl5nvvDaX7Pzss89Yvnw5xx13HMuWLeONN95g3rx5JCUlce211zJjxgyGDh3Kgw8+yNy5c6lfvz6///3vefzxx6Oro3mFgjho7nQRkVIiU3K///77fPHFF9HsOicnh1WrVjFnzhzGjh0bXYqz9PKhxZfsPOecczj33HNLHI+1ZGdkiVM4vCU7s7Ozoytezpkzh4ULF9KvX2jW0vz8fI499ljq16/P8uXLo0uVHjhwILrIi5coiAO6qyAiR4QqZsyJ8u233+L3+2nZsiXOOZ588klGjBhRos27775b4fKhSUlJtbZkZ+llRq+++mp+97vflWjzxhtvMHLkSF544YUqXftIo+gFGk4XEQnbtm0b119/PePGjcPMGDFiBM888wyFhYUAfPPNN+zdu5fhw4czdepU9u3bB5RdPvRIWbJz6NCh/P3vf2f79u0A7Nixg/Xr1zNo0CD+/e9/8+233wKwd+9eVq1aVe3vn2jKxEFBXETqtPz8fLKysigsLCQpKYnLL7+cW2+9FQgtE7p27Vr69OmDc44WLVrw5ptvMnLkSJYuXUq/fv1ISUnh7LPP5v77749e80hZsjMzM5N77rmHoUOHEgwGSU5OZvLkyfTv359nn32WMWPGcODAAQDuv/9+unTpUu19SKSELkWaCNW6FOmN95MRGMjwQXPpcsXvq+WaIiJV4bWlSCWxqroUqVJQUGGbiIh4Up0O4kZ4FELD6SIi4kF1OnodzL81Y5uIiHhPnQ7iRDJxn4bTRUTEe+p4EI/Q1yAiIt5Tp6NX9J64FkAREREPqtNBPEqFbSJSh0WWIu3Vqxd9+vRh3rx5tdaX4guoRNx4441kZWXRrVs36tWrF12GtHS7uqhOT/ZysDq9dvshIlKbii9F+t5773HnnXfy73//u5Z7ddBTTz0FwNq1azn33HPLzP4WUReWHi2tTqeg0dhtGk4XEQHYs2cPTZo0AUJTp5555pn06dOHzMxMZs2aFW33u9/9jhNPPJFhw4Zx8cUXR5cXXbhwIT179iQ7O5vx48fTo0cPgHKXNHXOMW7cOLp168Y555zD1q1bq9TfU089lbvuuovBgwczadIktmzZwgUXXEC/fv0YMGAAn332WfSzXHXVVQwYMIDevXvz1ltvAfDll1/Sv39/srKy6NmzZ3QaVq+oW/9kKSOSiSuIi0jte2jBQ6zYuaJar3li0xO5fcDtFbaJTLtaUFDA5s2bowuVpKWl8cYbb9CoUSO2b9/OwIEDOe+881i8eDGvvfYaS5YsoaioiD59+tC3b18Axo4dy5QpUxg0aFB0jXGAZ599NuaSpkuWLGHlypV8+eWXbNmyhW7dunH11VdX6TPu2bOHjz76CIAxY8bw61//moEDB0Yz92XLljFx4kRGjhzJtGnT2LVrFyeffDLDhg3j6aef5rbbbmPMmDHs378fr81iqiAOaDxdROqy4sPpn376KVdccQXLli3DOcdvfvMbPvroI3w+H5s2bWLLli188sknjB49mnr16gEwatQoAHbv3k1ubm50ec9LLrmE//u//wPKX9L0o48+4uKLL8bv99OmTRt+9KMfVbn/F110UfT1nDlzWLlyZXR7165d5Ofn8/777/POO+/w4IOhleIKCgqiC6Hcd999rFu3jgsuuIDOnTtX+f1rU50O4tHQ7avTdxVE5AhRWcZcE7Kzs9m+fTvbtm1j9uzZbNu2jcWLF5OcnEyHDh2iC5rEUlEWW96SprNnz65wSdN4lF56dMGCBaSkpJR5/zfffJPjjz++xP6uXbuSnZ3N22+/zbBhw3j++ecZPHjwYfWnJil6garTRUTCVqxYQSAQoFmzZuTk5NCyZUuSk5P58MMPWbduHRC6D/3WW29RUFBAXl4eb7/9NgBNmjQhPT09eh96xowZ0euWt6Tp4MGDmTFjBoFAgM2bN/Phhx8eVv+HDh0aLYQDoiMMI0aM4IknnojuX7JkCRBaO71z587cfPPNnHPOOXzxxReH9f41rU5n4mjudBGR6D1xCGWszz//PH6/n0svvZRRo0bRr18/srKyOPHEEwHo378/5513Hr169aJ9+/b069ePjIwMIHTv+5prrqFBgwacfvrp0f3lLWn64x//mA8++IDMzEy6du162GuKP/XUU/ziF7/gueeeo6ioiDPOOIOnnnqKe+65h1tuuYXMzEyCwSCdO3dm1qxZvPTSS7z88sskJyfTpk0b7rvvvsN6/5pWp5ci/eON95AWGMLws76hy+jrq+WaIiJV4dWlSPPy8mjYsCH79u1j8ODBTJkyhT59+kT3Azz44INs3ryZxx9/vJZ76x1VXYq0TmfiB2dsUyYuIlIV1157LcuXL6egoIArr7ySPn36APD222/zwAMPUFRURPv27Zk2bVrtdvQoV6eD+MHqdAVxEZGqeOmll2LuHzNmDGPGjKnh3tRdil6ge+IiIuJJdTp6HZyxTc+Ji4iI99TtIO5UnS4iIt6l6AWadlVERDypjgdxVaeLiESWIu3RowejRo1i9+7d1XLdtWvXRhdAqU6//e1vadu2bXRJ0uJztFe3pUuXMnv27IRd/3DV6eh1cClS3RMXkborMnf6smXLaNq0aYkZz45Uv/rVr1i6dClLly6Nzocej0AgUKX3URD3BA2ni4hAaO70TZs2AeUvRbp27VpOOukkrrnmGrp3787w4cPJz88HYPHixfTq1Yvs7OwS/xgoKChg7NixZGZm0rt37+j0qtOmTeP8889n1KhRdOzYkUmTJvHoo4/Su3dvBg4cyM6dO+Pu+9y5c+nduzeZmZlcffXV7N+/H4AOHTowceJETj31VF555RXWrFnDyJEj6du3L6eddhorVoRWjnvllVfo0aMHvXr1YvDgwRw4cIAJEyYwc+ZMsrKymDlz5uF/wdWsjj8nHqZMXESOAD/cfz/7v67epUhTTzqR1r/5TVxtA4EAc+fO5Wc/+xlQ/lKkAKtWreLll1/mL3/5Cz/96U957bXXuOyyyxg7dixPPvkkQ4YMYfz48dFrRwL6l19+yYoVKxg+fDjffPMNAMuWLWPJkiUUFBTQuXNnHnroIZYsWcKvfvUr/va3v3HLLbeU6euf/vQnXnzxRQAeeughhgwZwlVXXcXcuXPp2rUrV1xxBc8880z03LS0ND755BMAzjzzTCZPnkyXLl2YP38+N9xwAx988AETJ07kvffeo23btuzevZuUlBQmTpzIokWLmDRp0qF8/QlXpzNx09zpIiLRudObNWvGzp07GTZsGEB0KdKePXsydOjQ6FKkAB07dozOt963b1/Wrl1LTk4Ou3fvjs5/fvnll0ff45NPPolun3jiibRv3z4axM844wzS09Np0aIFGRkZ0aVNMzMzWbt2bcw+Fx9OHzFiBCtXrqRjx4507doVgCuvvDK6xjgQnYAmLy+PefPmceGFF5KVlcV1113H5s2bATjllFO46qqr+Mtf/lLlYffaUqczcT1iJiJHkngz5uoWuSeek5PDueeey1NPPcVNN93E9OnTYy5FCpCamho93+/3k5+fj3Ou3GVFK1qno/i1fD5fdNvn81FUVBTXZ6hsHZDIcqXBYJDGjRtHVzcrbvLkycyfP5+3336brKysmG2ONIpeoOp0EREgIyODJ554gkceeYTCwsJylyItT+PGjcnIyIgOW0+fPj16bPDgwdHtb775hvXr13PCCSdUW99PPPFE1q5dy+rVqwF44YUXYq6I1qhRIzp27Mgrr7wChIL/559/DsCaNWs4+eSTmThxIs2bN2fDhg2kp6eTm5tbbf2sbnU8eikTFxEprnfv3vTq1YsZM2Zw6aWXsmjRIvr168f06dOjS5FW5LnnnuPGG28kOzubevXqRfffcMMNBAIBMjMzGTNmDNOmTSuRgR+utLQ0nnvuOS688EIyMzPx+Xxcf33s1SmnT5/Os88+S69evejevXu0YG/8+PFkZmbSo0cPBg8eTK9evTjjjDNYvnz5EVvYVqeXIp30i9sxN4LhFxfQZcjZ1XJNEZGq8OpSpJIYVV2KtE6noBbjlYiIiFfU6SCOJnsREREPq+NBPEJBXEREvEdBXERExKMUxEVERDxKQVxERMSjFMRFROq4yFKkvXr1ok+fPsybN69ar3///fdX+Zxp06Yxbty4Evuee+656PKjKSkpZGZmJnwp0iNdQoO4mY00s5VmttrMynzLZpZqZjPDx+ebWYdE9kdERMqKTLv6+eef88ADD3DnnXdW6/UPJYjHMnbs2Oh86W3atOHDDz+MuRRpvFO1Hg0SFsTNzA88BZwFdAMuNrNupZr9DNjlnOsM/Al4KFH9ERGRyu3Zs4cmTZoAoSlJx48fT48ePcjMzIzOWFbe/s2bNzN48GCysrLo0aMHH3/8MXfccUd0gZVLL70UgBdffJEBAwZEFyCJLDby3HPP0bVrV4YMGcJ//vOfKvX77rvv5rrrrmPYsGGMHTuWoqIibr31VgYMGEDPnj3561//Gm374IMPRvdPnDgRgNzcXM466yx69epFjx49ePXVVw/vi6whiVwAZQCw2jn3LYCZzQBGA8uLtRkN/Db8+lVgkpmZ89o0ciIi1eDjv3/D9g151XrN5sc25LSfdq2wTSTIFhQUsHnzZj744AMAXn/99WiGvn37dvr378/gwYOZN29ezP0vvfQSI0aM4K677iIQCLBv3z5OO+00Jk2aFF1M5Ouvv2bmzJn85z//ITk5H9Sx1gAADEZJREFUmRtuuIHp06czbNgw7rnnHhYvXkxGRgZnnHEGvXv3rtJnXbJkCR999BFpaWk8/fTTtGzZkgULFrB//34GDhzI8OHDWbZsGevXr2f+/Pk45zj77LOZN28eGzZsoEOHDrzzzjsA5OTkHMK3XfMSGcTbAhuKbW8ETi6vjXOuyMxygGbA9uKNzOxa4FqA4447rto66GucSsqmr0nL6Ftt1xQR8ZrIcDrAp59+yhVXXMGyZcv45JNPuPjii/H7/bRq1YohQ4awcOHCcvf379+fq6++msLCQs4///zoUqXFzZ07l8WLF9O/f38g9A+Ili1b8v/bu//gqs46j+PvT2kwrWBXabFoasGx6laFmu1CkBlsjQsV22IdLLgtPxxsK7vqKFtmsltGUevUAR21K4is7TQ4YLGMthkLUxwF43QAm2ktbSmUWgub6kgI3c4W6S6Gr3+cE3obEnKSm9x7T/J5zdyZe899zj1fvtzkm+c5zzzP7t27ueKKK7jggguAZOvQzq1Ks5o9ezbV1dUAbNu2jaeffpp7770XSIrygQMH2LZtG1u3bj31B8LLL7/MM888w5QpU2hoaKChoYFrrrmGadOm9S+ZJTaYRby7FVS69rCztCEi1gHrIFk7vfjQEkvu+NpAfZSZWdF66zGXwtSpUzly5AhtbW09bu/Z0/Hp06fT3NzMgw8+yPz581m2bBkLFiw47dyFCxdyxx13vOb4/fff3+M2pll1bjfaeZ01a9ZQX1//mjZNTU0sX76cxYsXn3Z+S0sLW7ZsYdmyZVx99dX8R5m2hu2LwZzY1gpcVPC6BvhjT20knQ2cBxwdxJjMzOwM9u3bR0dHB2PGjGH69Ols2rSJjo4O2traaG5uZvLkyT0eP3jwIGPHjuWmm25i8eLFPProowBUVVVx4sQJAOrr69m8eTOHDx8G4OjRoxw8eJApU6awY8cO2tvbOXHixKmtQvtr5syZrFmz5tQkt/3793P8+HFmzpzJXXfdxbFjxwBobW3lyJEjvPDCC4waNYr58+ezdOnSU7FXusHsiT8CXCJpAvACMA/45y5tmoCFwE5gDvAr3w83MyutznvikPRgGxsbGTFiBNdddx07d+5k0qRJSGLlypVceOGFPR5vbGxk1apVVFVVMWrUKNavXw/AzTffzMSJE6mtrWXDhg3cfvvtzJgxg5MnT1JVVcXq1aupq6tjxYoVTJ06lXHjxlFbW3tqwlt/3HLLLRw6dOjUv2vs2LE88MADzJo1i3379lFXVwfA6NGj2bhxI3v37qWhoYGzzjqLkSNHsnbt2iKzWhqDuhWppFnAd4ARwN0R8XVJXwVaIqJJUjXwI+D9JD3weZ0T4XoykFuRmpmVm7citUJ93Yp0MHviRMQWYEuXY18qeP4K8InBjMHMzGyo8optZmZmOeUibmZmllMu4mZmZeb5vAb9+x64iJuZlVF1dTXt7e0u5MNcRNDe3n5qsZqsBnVim5mZnVlNTQ2tra20tbWVOxQrs+rqampqavp0jou4mVkZVVVVMWHChHKHYTnl4XQzM7OcchE3MzPLKRdxMzOznBrUZVcHg6Q24OAAfuT5dNn61PrFeSyec1g857B4zmHxBjqHF0fEBd29kbsiPtAktfS0Jq1l5zwWzzksnnNYPOeweKXMoYfTzczMcspF3MzMLKdcxGFduQMYIpzH4jmHxXMOi+ccFq9kORz298TNzMzyyj1xMzOznBo2RVzSVZL2S3pWUkM3779O0qb0/d2Sxpc+ysqWIYdLJe2VtEfSLyVdXI44K1lvOSxoN0dSSPIs4W5kyaOk69Pv41OSNpY6xkqX4ef5bZK2S3os/ZmeVY44K5WkuyUdlvRkD+9L0p1pfvdIqh2UQCJiyD+AEcDvgbcDI4HHgUu7tPkXYG36fB6wqdxxV9IjYw6vBM5Nny9xDvuew7TdaKAZ2AVcXu64K+2R8bt4CfAY8Mb09dhyx11Jj4w5XAcsSZ9fCjxf7rgr6QFMB2qBJ3t4fxawFRBQB+wejDiGS098MvBsRDwXEf8P3AvM7tJmNtCYPt8M1EtSCWOsdL3mMCK2R8Rf0pe7gL5txzP0ZfkeAnwNWAm8UsrgciRLHm8CVkfEiwARcbjEMVa6LDkM4A3p8/OAP5YwvooXEc3A0TM0mQ2sj8Qu4O8kjRvoOIZLEX8r8N8Fr1vTY922iYi/Ai8BY0oSXT5kyWGhxSR/hdqres2hpPcDF0XEz0sZWM5k+S6+E3inpIcl7ZJ0Vcmiy4csOVwB3CipFdgCfK40oQ0Zff2d2S/DZSvS7nrUXaflZ2kznGXOj6QbgcuBDw5qRPlzxhxKOgv4NrCoVAHlVJbv4tkkQ+pXkIwI/UbSeyPifwY5trzIksNPAvdExLckTQV+lObw5OCHNySUpKYMl554K3BRwesaTh8aOtVG0tkkw0dnGioZbrLkEEkfBm4Dro2I/ytRbHnRWw5HA+8Fdkh6nuQ+WpMnt50m68/zAxFxIiL+AOwnKeqWyJLDxcBPACJiJ1BNsia4ZZPpd2axhksRfwS4RNIESSNJJq41dWnTBCxMn88BfhXp7AQDMuQwHQr+AUkB9z3I050xhxHxUkScHxHjI2I8ybyCayOipTzhVqwsP8/3k0y0RNL5JMPrz5U0ysqWJYeHgHoASX9PUsTbShplvjUBC9JZ6nXASxHxp4G+yLAYTo+Iv0r6LPAQyazMuyPiKUlfBVoiogm4i2S46FmSHvi88kVceTLmcBUwCrgvnRN4KCKuLVvQFSZjDq0XGfP4EDBD0l6gA1gWEe3li7qyZMzhvwH/JemLJMPAi9yxeZWkH5Pcrjk/nTfwZaAKICLWkswjmAU8C/wF+NSgxOH/EzMzs3waLsPpZmZmQ46LuJmZWU65iJuZmeWUi7iZmVlOuYibmZnllIu4WYlJ6pD0u4LH+DO0Hd/TLkl9vOaOdMeqx9OlSN/Vj8/4jKQF6fNFkt5S8N4PJV06wHE+IumyDOd8QdK5xV7bLI9cxM1K73hEXFbweL5E170hIiaRbPSzqq8nR8TaiFifvlwEvKXgvU9HxN4BifLVONeQLc4vAC7iNiy5iJtVgLTH/RtJj6aPD3TT5j2Sfpv23vdIuiQ9fmPB8R9IGtHL5ZqBd6Tn1qf7RT+R7o/8uvT4N/Tq3vDfTI+tkHSrpDkka+NvSK95TtqDvlzSEkkrC2JeJOk/+xnnTgo2jJD0fUktSvYH/0p67PMkf0xsl7Q9PTZD0s40j/dJGtXLdcxyy0XcrPTOKRhK/1l67DDwTxFRC8wF7uzmvM8A342Iy0iKaGu6HOZcYFp6vAO4oZfrXwM8IakauAeYGxHvI1nBcYmkNwHXAe+JiInA7YUnR8RmoIWkx3xZRBwveHsz8PGC13OBTf2M8yqS5VM73RYRlwMTgQ9KmhgRd5KsR31lRFyZLrG6HPhwmssWYGkv1zHLrWGx7KpZhTmeFrJCVcD30nvAHSRrfXe1E7hNUg3w04g4IKke+AfgkXSp23NI/iDozgZJx4HnSbaVfBfwh4h4Jn2/EfhX4Hske5n/UNKDQOZtUSOiTdJz6VrRB9JrPJx+bl/ifD3JcqC1Bcevl3Qzye+tccClwJ4u59alxx9OrzOSJG9mQ5KLuFll+CLwZ2ASyQjZK10bRMRGSbuBjwIPSfo0yXaHjRHx7xmucUPhZiqSxnTXKF1XezLJ5hfzgM8CH+rDv2UTcD2wD/hZRISSipo5TuBx4BvAauDjkiYAtwL/GBEvSrqHZEOOrgT8IiI+2Yd4zXLLw+lmleE84E/pXs3zSXqhryHp7cBz6RByE8mw8i+BOZLGpm3eJOnijNfcB4yX9I709Xzg1+k95PMiYgvJpLHuZoj/L8nWqd35KfAxkv2oN6XH+hRnRJwgGRavS4fi3wAcA16S9GbgIz3EsguY1vlvknSupO5GNcyGBBdxs8qwBlgoaRfJUPqxbtrMBZ6U9Dvg3cD6dEb4cmCbpD3AL0iGmnsVEa+Q7Kx0n6QngJPAWpKC+PP0835NMkrQ1T3A2s6JbV0+90VgL3BxRPw2PdbnONN77d8Cbo2Ix4HHgKeAu0mG6DutA7ZK2h4RbSQz53+cXmcXSa7MhiTvYmZmZpZT7ombmZnllIu4mZlZTrmIm5mZ5ZSLuJmZWU65iJuZmeWUi7iZmVlOuYibmZnllIu4mZlZTv0NZNMRGzEstO4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8,6))\n",
    "plt.plot(LR_roc[0],LR_roc[1],label=\"Logistic Regression\")\n",
    "plt.plot(DT_roc[0],DT_roc[1],label=\"Decision Tree\")\n",
    "plt.plot(BaT_roc[0],BaT_roc[1],label=\"Bagged Trees\")\n",
    "plt.plot(RF_roc[0],RF_roc[1],label=\"Random Forest\")\n",
    "plt.plot(BoT_roc[0],BoT_roc[1],label=\"Boosted Trees\")\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.legend()\n",
    "plt.title('ROC Curve')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " Boosted trees performed the best and decision tree performed the worst"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculating the AUC score of these 5 classifiers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# AUC values\n",
    "LR_auc=roc_auc_score(y_holdout, LR_yprob)\n",
    "DT_auc=roc_auc_score(y_holdout, DT_yprob)\n",
    "BaT_auc=roc_auc_score(y_holdout, BaT_yprob)\n",
    "RF_auc=roc_auc_score(y_holdout, RF_yprob)\n",
    "BoT_auc=roc_auc_score(y_holdout,BoT_yprob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Classification model</th>\n",
       "      <th>AUC Scores</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Boosted Trees</td>\n",
       "      <td>0.973840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>0.972617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Bagged Trees</td>\n",
       "      <td>0.967590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Random Forest</td>\n",
       "      <td>0.964048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Decision Tree</td>\n",
       "      <td>0.927321</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Classification model  AUC Scores\n",
       "4        Boosted Trees    0.973840\n",
       "0  Logistic Regression    0.972617\n",
       "2         Bagged Trees    0.967590\n",
       "3        Random Forest    0.964048\n",
       "1        Decision Tree    0.927321"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_AUC=[['Logistic Regression',LR_auc],['Decision Tree',DT_auc],['Bagged Trees',BaT_auc],['Random Forest',RF_auc],['Boosted Trees',BoT_auc]]\n",
    "AUC_scores=pd.DataFrame(data_AUC,columns=[\"Classification model\",\"AUC Scores\"])\n",
    "AUC_scores.sort_values(by='AUC Scores',ascending= False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Boosted tree has the highest AUC score, followed by Logistic and decision tree is the worst."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lastly, with the best classifier (based on the observations from above), present the confusion matrix on the holdout set.\n",
    "\n",
    "Logistic and boosted seems be performing well. We will consider logistic regression to be best classifier and present the confusion matrix on the holdout set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix on the test set \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>747</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>25</td>\n",
       "      <td>122</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0    1\n",
       "0  747    4\n",
       "1   25  122"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Confusion matrix\n",
    "LR_yhat=np.where(LR_yprob > 0.5,1,0)\n",
    "LR_confmat=pd.DataFrame(confusion_matrix(y_holdout,LR_yhat))\n",
    "print(\"Confusion matrix on the test set \")\n",
    "LR_confmat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}