neural_net_pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import accuracy_score, roc_auc_score, recall_score\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from imblearn.pipeline import Pipeline\n",
    "\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_state = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('dataset_diabetes/diabetic_data_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Removing several medication columns also\n",
    "remove_cols = ['encounter_id', 'patient_nbr', 'weight', 'payer_code', 'medical_specialty', 'examide', \n",
    "    'citoglipton', 'chlorpropamide', 'acetohexamide', 'tolbutamide', 'acarbose', 'miglitol', \n",
    "    'troglitazone', 'tolazamide', 'glyburide-metformin', 'glipizide-metformin', 'glimepiride-pioglitazone', \n",
    "    'metformin-rosiglitazone', 'metformin-pioglitazone']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = df_train[[col for col in df_train.columns if col not in remove_cols]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['race', 'gender', 'age', 'admission_type_id',\n",
       "       'discharge_disposition_id', 'admission_source_id', 'time_in_hospital',\n",
       "       'num_lab_procedures', 'num_procedures', 'num_medications',\n",
       "       'number_outpatient', 'number_emergency', 'number_inpatient', 'diag_1',\n",
       "       'diag_2', 'diag_3', 'number_diagnoses', 'max_glu_serum', 'A1Cresult',\n",
       "       'metformin', 'repaglinide', 'nateglinide', 'glimepiride', 'glipizide',\n",
       "       'glyburide', 'pioglitazone', 'rosiglitazone', 'insulin', 'change',\n",
       "       'diabetesMed', 'readmitted'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('NO', 0.5393799439886012),\n",
       " ('>30', 0.3491868520611212),\n",
       " ('<30', 0.1114332039502776)]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(key, val/len(df_train.readmitted)) for (key, val) in Counter(df_train.readmitted).most_common()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def binarize_readmitted(x):\n",
    "    if x in ['<30']: # readmitted in less than 30 days is positive class\n",
    "        return 1 \n",
    "    else:\n",
    "        return 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['label'] = df_train.apply(lambda x: binarize_readmitted(x['readmitted']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 0.8885667960497224), (1, 0.1114332039502776)]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(key, val/len(df_train.readmitted)) for (key, val) in Counter(df_train['label']).most_common()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def map_age(x):\n",
    "    if x == '[0-10)':\n",
    "        return 5\n",
    "    elif x == '[10-20)':\n",
    "        return 15\n",
    "    elif x == '[20-30)':\n",
    "        return 25\n",
    "    elif x == '[30-40)':\n",
    "        return 35\n",
    "    elif x == '[40-50)':\n",
    "        return 45\n",
    "    elif x == '[50-60)':\n",
    "        return 55\n",
    "    elif x == '[60-70)':\n",
    "        return 65\n",
    "    elif x == '[70-80)':\n",
    "        return 75\n",
    "    elif x == '[80-90)':\n",
    "        return 85\n",
    "    else:\n",
    "        return 95\n",
    "\n",
    "admission_dict = {1: 'Emergency', 2: 'Urgent', 3: 'Elective', 4: 'Newborn', 7: 'Trauma Center'}\n",
    "def map_admission(x):\n",
    "    if x in [5,6,8]:\n",
    "        return 'Not Available'\n",
    "    elif x == 4:\n",
    "        return 'Elective'\n",
    "    else:\n",
    "        return admission_dict[x]\n",
    "    \n",
    "def discharged_home(x):\n",
    "    if x == 1:\n",
    "        return 1 # Discharged home\n",
    "    else:\n",
    "        return 0 # Other \n",
    "\n",
    "def map_source(x):\n",
    "    if x in [1,2,3]:\n",
    "        return 'Referral' \n",
    "    elif x == 7:\n",
    "        return 'Emergency Room'\n",
    "    else:\n",
    "        return 'Other'\n",
    "\n",
    "def map_diag(x):\n",
    "    if x[0].isdigit() == False:\n",
    "        return 'other'\n",
    "    \n",
    "    float_x = float(x)\n",
    "    if (float_x >= 390 and float_x <= 459) or float_x == 785: # 390–459, 785\n",
    "        return 'circulatory'\n",
    "    elif (float_x >= 460 and float_x <= 519) or float_x == 786: # 460–519, 786\n",
    "        return 'respiratory'\n",
    "    elif (float_x >= 520 and float_x <= 579) or float_x == 787: # 520–579, 787\n",
    "        return 'digestive'\n",
    "    elif float_x >= 250 and float_x < 251: # 250.xx\n",
    "        return 'diabetes'\n",
    "    elif float_x >= 800 and float_x <= 999: # 800–999\n",
    "        return 'injury'\n",
    "    elif float_x >= 710 and float_x <= 739: # 710–739\n",
    "        return 'musculoskeletal'\n",
    "    elif (float_x >= 580 and float_x <= 629) or float_x == 788: # 580–629, 788\n",
    "        return 'genitourinary'\n",
    "    elif float_x >= 140 and float_x <= 239: # 140–239\n",
    "        return 'neoplasms'\n",
    "    else:\n",
    "        return 'other'\n",
    "\n",
    "def binarize_yn(x):\n",
    "    if x in ['Yes', 'Ch']: \n",
    "        return 1 \n",
    "    else:\n",
    "        return 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['age_processed'] = df_train.apply(lambda x: map_age(x['age']), axis=1)\n",
    "df_train['admission_type_processed'] = df_train.apply(lambda x: map_admission(x['admission_type_id']), axis=1)\n",
    "df_train['discharged_processed'] = df_train.apply(lambda x: discharged_home(x['discharge_disposition_id']), axis=1)\n",
    "df_train['source_processed'] = df_train.apply(lambda x: map_source(x['admission_source_id']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['diag1_processed'] = df_train.apply(lambda x: map_diag(x['diag_1']), axis=1)\n",
    "df_train['diag2_processed'] = df_train.apply(lambda x: map_diag(x['diag_2']), axis=1)\n",
    "df_train['diag3_processed'] = df_train.apply(lambda x: map_diag(x['diag_3']), axis=1)\n",
    "df_train['change_processed'] = df_train.apply(lambda x: binarize_yn(x['change']), axis=1)\n",
    "df_train['diabetesMed_processed'] = df_train.apply(lambda x: binarize_yn(x['diabetesMed']), axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## One Hot Encodings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "ohe_cols = ['admission_type_processed', 'source_processed',\n",
    "    'diag1_processed', 'diag2_processed', 'diag3_processed', 'race', 'gender', \n",
    "    'max_glu_serum', 'A1Cresult', 'metformin', 'repaglinide', 'nateglinide', \n",
    "    'glimepiride', 'glipizide', 'glyburide', 'pioglitazone', 'rosiglitazone', 'insulin']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81410\n"
     ]
    }
   ],
   "source": [
    "# Remove 2 rows where gender is invalid (probably doesn't matter but just incase)\n",
    "df_train = df_train[df_train.gender != 'Unknown/Invalid']\n",
    "print(len(df_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_sub = df_train[ohe_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "ohe = OneHotEncoder(categories='auto')\n",
    "train_ohe = ohe.fit_transform(df_train_sub).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# obtain column names for one hot encoded columns \n",
    "feature_labels = []\n",
    "feature_labels_ = ohe.categories_\n",
    "for col, values in zip(df_train_sub.columns, feature_labels_):\n",
    "    for val in values:\n",
    "        feature_labels.append(col+'_'+val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_ohe = pd.DataFrame(train_ohe, columns = feature_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_cols = ['age_processed', 'discharged_processed', 'change_processed', 'diabetesMed_processed', \n",
    "        'time_in_hospital', 'num_lab_procedures', 'num_procedures', 'num_medications', \n",
    "        'number_outpatient','number_emergency', 'number_inpatient', 'number_diagnoses']\n",
    "\n",
    "df_train_num = df_train[num_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "df_train_num_scaled = scaler.fit_transform(df_train_num.astype(np.float64))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_num = pd.DataFrame(df_train_num_scaled, index=df_train_num.index, columns=df_train_num.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(81410, 87) (81410, 12)\n"
     ]
    }
   ],
   "source": [
    "print(df_train_ohe.shape, df_train_num.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reindexing because df_train_ohe has different indices than df_train_num (due to removing the two invalid gender entries)\n",
    "# and not doing this will create null entries when concatenating \n",
    "df_train_ohe.reset_index(drop=True, inplace=True)\n",
    "df_train_num.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = pd.concat([df_train_ohe, df_train_num], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "admission_type_processed_Elective\n",
      "Counter({0.0: 66348, 1.0: 15062}) \n",
      "\n",
      "admission_type_processed_Emergency\n",
      "Counter({1.0: 43216, 0.0: 38194}) \n",
      "\n",
      "admission_type_processed_Not Available\n",
      "Counter({0.0: 73136, 1.0: 8274}) \n",
      "\n",
      "admission_type_processed_Trauma Center\n",
      "Counter({0.0: 81397, 1.0: 13}) \n",
      "\n",
      "admission_type_processed_Urgent\n",
      "Counter({0.0: 66565, 1.0: 14845}) \n",
      "\n",
      "source_processed_Emergency Room\n",
      "Counter({1.0: 45988, 0.0: 35422}) \n",
      "\n",
      "source_processed_Other\n",
      "Counter({0.0: 70656, 1.0: 10754}) \n",
      "\n",
      "source_processed_Referral\n",
      "Counter({0.0: 56742, 1.0: 24668}) \n",
      "\n",
      "diag1_processed_circulatory\n",
      "Counter({0.0: 56981, 1.0: 24429}) \n",
      "\n",
      "diag1_processed_diabetes\n",
      "Counter({0.0: 74423, 1.0: 6987}) \n",
      "\n",
      "diag1_processed_digestive\n",
      "Counter({0.0: 73861, 1.0: 7549}) \n",
      "\n",
      "diag1_processed_genitourinary\n",
      "Counter({0.0: 77288, 1.0: 4122}) \n",
      "\n",
      "diag1_processed_injury\n",
      "Counter({0.0: 75921, 1.0: 5489}) \n",
      "\n",
      "diag1_processed_musculoskeletal\n",
      "Counter({0.0: 77481, 1.0: 3929}) \n",
      "\n",
      "diag1_processed_neoplasms\n",
      "Counter({0.0: 78671, 1.0: 2739}) \n",
      "\n",
      "diag1_processed_other\n",
      "Counter({0.0: 66774, 1.0: 14636}) \n",
      "\n",
      "diag1_processed_respiratory\n",
      "Counter({0.0: 69880, 1.0: 11530}) \n",
      "\n",
      "diag2_processed_circulatory\n",
      "Counter({0.0: 55954, 1.0: 25456}) \n",
      "\n",
      "diag2_processed_diabetes\n",
      "Counter({0.0: 71193, 1.0: 10217}) \n",
      "\n",
      "diag2_processed_digestive\n",
      "Counter({0.0: 78092, 1.0: 3318}) \n",
      "\n",
      "diag2_processed_genitourinary\n",
      "Counter({0.0: 74693, 1.0: 6717}) \n",
      "\n",
      "diag2_processed_injury\n",
      "Counter({0.0: 79441, 1.0: 1969}) \n",
      "\n",
      "diag2_processed_musculoskeletal\n",
      "Counter({0.0: 80012, 1.0: 1398}) \n",
      "\n",
      "diag2_processed_neoplasms\n",
      "Counter({0.0: 79371, 1.0: 2039}) \n",
      "\n",
      "diag2_processed_other\n",
      "Counter({0.0: 59793, 1.0: 21617}) \n",
      "\n",
      "diag2_processed_respiratory\n",
      "Counter({0.0: 72731, 1.0: 8679}) \n",
      "\n",
      "diag3_processed_circulatory\n",
      "Counter({0.0: 57251, 1.0: 24159}) \n",
      "\n",
      "diag3_processed_diabetes\n",
      "Counter({0.0: 67651, 1.0: 13759}) \n",
      "\n",
      "diag3_processed_digestive\n",
      "Counter({0.0: 78281, 1.0: 3129}) \n",
      "\n",
      "diag3_processed_genitourinary\n",
      "Counter({0.0: 76056, 1.0: 5354}) \n",
      "\n",
      "diag3_processed_injury\n",
      "Counter({0.0: 79851, 1.0: 1559}) \n",
      "\n",
      "diag3_processed_musculoskeletal\n",
      "Counter({0.0: 79893, 1.0: 1517}) \n",
      "\n",
      "diag3_processed_neoplasms\n",
      "Counter({0.0: 79917, 1.0: 1493}) \n",
      "\n",
      "diag3_processed_other\n",
      "Counter({0.0: 56835, 1.0: 24575}) \n",
      "\n",
      "diag3_processed_respiratory\n",
      "Counter({0.0: 75545, 1.0: 5865}) \n",
      "\n",
      "race_?\n",
      "Counter({0.0: 79588, 1.0: 1822}) \n",
      "\n",
      "race_AfricanAmerican\n",
      "Counter({0.0: 66085, 1.0: 15325}) \n",
      "\n",
      "race_Asian\n",
      "Counter({0.0: 80893, 1.0: 517}) \n",
      "\n",
      "race_Caucasian\n",
      "Counter({1.0: 60871, 0.0: 20539}) \n",
      "\n",
      "race_Hispanic\n",
      "Counter({0.0: 79760, 1.0: 1650}) \n",
      "\n",
      "race_Other\n",
      "Counter({0.0: 80185, 1.0: 1225}) \n",
      "\n",
      "gender_Female\n",
      "Counter({1.0: 43740, 0.0: 37670}) \n",
      "\n",
      "gender_Male\n",
      "Counter({0.0: 43740, 1.0: 37670}) \n",
      "\n",
      "max_glu_serum_>200\n",
      "Counter({0.0: 80209, 1.0: 1201}) \n",
      "\n",
      "max_glu_serum_>300\n",
      "Counter({0.0: 80397, 1.0: 1013}) \n",
      "\n",
      "max_glu_serum_None\n",
      "Counter({1.0: 77128, 0.0: 4282}) \n",
      "\n",
      "max_glu_serum_Norm\n",
      "Counter({0.0: 79342, 1.0: 2068}) \n",
      "\n",
      "A1Cresult_>7\n",
      "Counter({0.0: 78419, 1.0: 2991}) \n",
      "\n",
      "A1Cresult_>8\n",
      "Counter({0.0: 74810, 1.0: 6600}) \n",
      "\n",
      "A1Cresult_None\n",
      "Counter({1.0: 67776, 0.0: 13634}) \n",
      "\n",
      "A1Cresult_Norm\n",
      "Counter({0.0: 77367, 1.0: 4043}) \n",
      "\n",
      "metformin_Down\n",
      "Counter({0.0: 80955, 1.0: 455}) \n",
      "\n",
      "metformin_No\n",
      "Counter({1.0: 65383, 0.0: 16027}) \n",
      "\n",
      "metformin_Steady\n",
      "Counter({0.0: 66707, 1.0: 14703}) \n",
      "\n",
      "metformin_Up\n",
      "Counter({0.0: 80541, 1.0: 869}) \n",
      "\n",
      "repaglinide_Down\n",
      "Counter({0.0: 81375, 1.0: 35}) \n",
      "\n",
      "repaglinide_No\n",
      "Counter({1.0: 80160, 0.0: 1250}) \n",
      "\n",
      "repaglinide_Steady\n",
      "Counter({0.0: 80284, 1.0: 1126}) \n",
      "\n",
      "repaglinide_Up\n",
      "Counter({0.0: 81321, 1.0: 89}) \n",
      "\n",
      "nateglinide_Down\n",
      "Counter({0.0: 81402, 1.0: 8}) \n",
      "\n",
      "nateglinide_No\n",
      "Counter({1.0: 80843, 0.0: 567}) \n",
      "\n",
      "nateglinide_Steady\n",
      "Counter({0.0: 80871, 1.0: 539}) \n",
      "\n",
      "nateglinide_Up\n",
      "Counter({0.0: 81390, 1.0: 20}) \n",
      "\n",
      "glimepiride_Down\n",
      "Counter({0.0: 81257, 1.0: 153}) \n",
      "\n",
      "glimepiride_No\n",
      "Counter({1.0: 77253, 0.0: 4157}) \n",
      "\n",
      "glimepiride_Steady\n",
      "Counter({0.0: 77672, 1.0: 3738}) \n",
      "\n",
      "glimepiride_Up\n",
      "Counter({0.0: 81144, 1.0: 266}) \n",
      "\n",
      "glipizide_Down\n",
      "Counter({0.0: 80951, 1.0: 459}) \n",
      "\n",
      "glipizide_No\n",
      "Counter({1.0: 71344, 0.0: 10066}) \n",
      "\n",
      "glipizide_Steady\n",
      "Counter({0.0: 72426, 1.0: 8984}) \n",
      "\n",
      "glipizide_Up\n",
      "Counter({0.0: 80787, 1.0: 623}) \n",
      "\n",
      "glyburide_Down\n",
      "Counter({0.0: 80954, 1.0: 456}) \n",
      "\n",
      "glyburide_No\n",
      "Counter({1.0: 72903, 0.0: 8507}) \n",
      "\n",
      "glyburide_Steady\n",
      "Counter({0.0: 74004, 1.0: 7406}) \n",
      "\n",
      "glyburide_Up\n",
      "Counter({0.0: 80765, 1.0: 645}) \n",
      "\n",
      "pioglitazone_Down\n",
      "Counter({0.0: 81321, 1.0: 89}) \n",
      "\n",
      "pioglitazone_No\n",
      "Counter({1.0: 75592, 0.0: 5818}) \n",
      "\n",
      "pioglitazone_Steady\n",
      "Counter({0.0: 75861, 1.0: 5549}) \n",
      "\n",
      "pioglitazone_Up\n",
      "Counter({0.0: 81230, 1.0: 180}) \n",
      "\n",
      "rosiglitazone_Down\n",
      "Counter({0.0: 81342, 1.0: 68}) \n",
      "\n",
      "rosiglitazone_No\n",
      "Counter({1.0: 76323, 0.0: 5087}) \n",
      "\n",
      "rosiglitazone_Steady\n",
      "Counter({0.0: 76535, 1.0: 4875}) \n",
      "\n",
      "rosiglitazone_Up\n",
      "Counter({0.0: 81266, 1.0: 144}) \n",
      "\n",
      "insulin_Down\n",
      "Counter({0.0: 71617, 1.0: 9793}) \n",
      "\n",
      "insulin_No\n",
      "Counter({0.0: 43481, 1.0: 37929}) \n",
      "\n",
      "insulin_Steady\n",
      "Counter({0.0: 56818, 1.0: 24592}) \n",
      "\n",
      "insulin_Up\n",
      "Counter({0.0: 72314, 1.0: 9096}) \n",
      "\n",
      "age_processed\n",
      "Counter({0.5670535328754747: 20878, -0.05955089631014834: 17958, -0.6861553254957714: 13741, 1.1936579620610976: 13737, -1.3127597546813945: 7810, -1.9393641838670175: 3030, 1.8202623912467208: 2239, -2.5659686130526405: 1335, -3.1925730422382634: 551, -3.8191774714238864: 131}) \n",
      "\n",
      "discharged_processed\n",
      "Counter({0.8282276406317545: 48287, -1.2073975208521428: 33123}) \n",
      "\n",
      "change_processed\n",
      "Counter({-0.9277013987634344: 43754, 1.0779330518773982: 37656}) \n",
      "\n",
      "diabetesMed_processed\n",
      "Counter({0.5478566667504611: 62616, -1.8252949369610973: 18794}) \n",
      "\n",
      "time_in_hospital\n",
      "Counter({-0.46815377146633436: 14169, -0.8021888340453177: 13819, -1.1362238966243012: 11375, -0.13411870888735095: 11087, 0.19991635369163244: 7926, 0.5339514162706158: 6033, 0.8679864788495992: 4704, 1.2020215414285826: 3504, 1.536056604007566: 2425, 1.8700916665865495: 1890, 2.204126729165533: 1476, 2.5381617917445163: 1157, 2.8721968543234997: 1004, 3.206231916902483: 841}) \n",
      "\n",
      "num_lab_procedures\n",
      "Counter({-2.1319629586918363: 2598, -0.0039371235057667: 2208, 0.046730158284377814: 1984, 0.09739744007452232: 1905, -0.25727353245648926: 1785, -0.15593896887620023: 1775, 0.14806472186466685: 1725, -0.3079408142466338: 1695, -0.10527168708605572: 1685, 0.19873200365481136: 1674, -0.054604405295911214: 1661, 0.24939928544495585: 1647, -0.20660625066634475: 1647, 0.3000665672351004: 1630, -0.3586080960367783: 1587, -0.4092753778269228: 1541, 0.3507338490252449: 1538, 0.4014011308153894: 1528, 0.5534029761858229: 1509, 0.45206841260553393: 1506, 0.6547375397661119: 1463, 0.6040702579759675: 1460, 0.5027356943956784: 1439, 0.7054048215562565: 1400, 0.756072103346401: 1370, -0.4599426596170673: 1368, 0.9080739487168346: 1318, 0.85740666692669: 1286, 0.8067393851365455: 1278, 1.0094085122971235: 1136, 0.958741230506979: 1130, 1.060075794087268: 1091, -0.6626117867776453: 1077, -0.7132790685677899: 1060, -0.6119445049875009: 1047, -0.5612772231973563: 1045, -0.5106099414072118: 1024, 1.1614103576675572: 984, 1.1107430758774126: 972, -2.0812956769016915: 901, -0.8652809139382234: 875, 1.2120776394577015: 872, -0.9159481957283679: 848, 1.262744921247846: 841, -0.7639463503579343: 824, 1.3134122030379907: 794, -1.219951886469235: 769, -1.7266247043706802: 754, -0.8146136321480789: 680, -1.6759574225805356: 675, 1.3640794848281352: 666, -1.0172827593086569: 657, 1.4147467666182796: 614, -1.1692846046790906: 605, -1.321286450049524: 605, 1.4654140484084242: 604, -0.9666154775185124: 602, -1.0679500410988014: 582, -1.2706191682593795: 572, -1.625290140790391: 543, -1.118617322888946: 537, -2.030628395111547: 533, 1.5160813301985687: 520, 1.566748611988713: 503, -1.3719537318396686: 433, 1.6174158937788576: 431, -1.5746228590002467: 407, 1.6680831755690022: 377, -1.422621013629813: 363, -1.523955577210102: 343, 1.7187504573591468: 338, -1.9799611133214026: 315, 1.7694177391492911: 293, -1.7772919861608247: 283, -1.4732882954199575: 266, -1.827959267950969: 266, 1.8200850209394357: 260, 1.8707523027295803: 252, -1.9292938315312582: 233, 1.9214195845197248: 225, -1.8786265497411136: 218, 2.022754148100014: 191, 1.9720868663098692: 185, 2.0734214298901583: 127, 2.1240887116803027: 126, 2.1747559934704475: 106, 2.276090557050736: 82, 2.225423275260592: 74, 2.326757838840881: 62, 2.3774251206310253: 51, 2.42809240242117: 50, 2.5294269660014588: 47, 2.4787596842113144: 41, 2.5800942477916036: 39, 2.630761529581748: 38, 2.732096093162037: 27, 2.7827633749521814: 20, 2.6814288113718923: 19, 2.934765220322615: 11, 2.8840979385324705: 9, 2.833430656742326: 8, 2.9854325021127597: 7, 3.036099783902904: 5, 3.1881016292733375: 5, 3.2894361928536267: 4, 3.340103474643771: 4, 3.137434347483193: 4, 3.0867670656930484: 3, 3.5934398835944936: 2, 3.9481108561255054: 1, 3.238768911063482: 1, 3.8974435743353606: 1, 4.5054509558170945: 1, 3.5427726018043493: 1, 3.796109010755072: 1, 3.44143803822406: 1, 4.353449110446661: 1}) \n",
      "\n",
      "num_procedures\n",
      "Counter({-0.784868110556935: 37349, -0.19831546099122993: 16624, 0.3882371885744751: 10131, 0.9747898381401802: 7534, 2.734447786837295: 3928, 1.561342487705885: 3341, 2.14789513727159: 2503}) \n",
      "\n",
      "num_medications\n",
      "Counter({-0.3705251967129147: 4879, -0.49368865925241173: 4816, -0.6168521217919088: 4669, -0.1241982716339205: 4611, -0.24736173417341759: 4593, -0.0010348090944234136: 4351, -0.7400155843314059: 4259, 0.12212865344507366: 3945, -0.863179046870903: 3903, 0.24529211598457074: 3626, -0.9863425094104: 3511, 0.36845557852406785: 3289, 0.4916190410635649: 2976, -1.1095059719498972: 2753, 0.614782503603062: 2547, 0.737945966142559: 2282, -1.2326694344893943: 2196, 0.8611094286820562: 1906, 0.9842728912215533: 1698, -1.3558328970288913: 1618, 1.1074363537610503: 1504, 1.2305998163005474: 1290, -1.4789963595683884: 1136, 1.3537632788400444: 1132, 1.4769267413795415: 966, 1.6000902039190386: 780, -1.6021598221078854: 716, 1.7232536664585356: 685, 1.846417128998033: 570, 1.96958059153753: 503, 2.092744054077027: 386, -1.7253232846473825: 363, 2.215907516616524: 355, 2.339070979156021: 317, 2.4622344416955184: 238, 2.5853979042350153: 234, -1.8484867471868796: 219, 2.7085613667745125: 196, 2.8317248293140094: 165, 2.9548882918535067: 150, 3.0780517543930035: 109, 3.3243786794719976: 108, 3.2012152169325008: 97, 3.693869067090489: 76, 3.5707056045509917: 74, 3.447542142011495: 72, 3.8170325296299863: 61, 4.063359454708981: 50, 3.940195992169483: 50, 4.186522917248478: 45, 4.432849842327472: 44, 4.3096863797879745: 33, 4.556013304866969: 29, 4.92550369248546: 29, 4.679176767406466: 27, 4.802340229945963: 25, 5.048667155024957: 23, 5.171830617564455: 20, 5.294994080103952: 15, 5.541321005182946: 13, 5.418157542643448: 13, 5.78764793026194: 12, 5.664484467722443: 10, 6.033974855340934: 9, 6.403465242959426: 6, 6.280301780419928: 5, 6.157138317880431: 5, 5.9108113928014365: 4, 6.896119093117414: 3, 6.526628705498922: 3, 7.265609480735905: 2, 6.649792168038419: 2, 8.004590255972888: 1, 7.758263330893893: 1, 7.142446018196408: 1}) \n",
      "\n",
      "number_outpatient\n",
      "Counter({-0.2911746303454171: 68018, 0.4997935920317194: 6865, 1.290761814408856: 2872, 2.0817300367859923: 1620, 2.872698259163129: 888, 3.663666481540265: 423, 4.454634703917401: 228, 5.245602926294538: 130, 6.036571148671674: 72, 6.827539371048812: 71, 7.618507593425948: 41, 8.409475815803084: 35, 9.991412260557357: 24, 9.20044403818022: 23, 10.782380482934494: 22, 11.57334870531163: 15, 12.364316927688767: 13, 17.110126261951585: 5, 16.31915803957445: 5, 13.94625337244304: 5, 15.528189817197312: 5, 13.155285150065904: 5, 14.737221594820175: 3, 21.064967373837266: 3, 18.69206270670586: 2, 25.810776708100082: 2, 17.901094484328723: 2, 27.392713152854355: 2, 20.27399915146013: 2, 26.60174493047722: 1, 22.64690381859154: 1, 28.183681375231494: 1, 21.855935596214405: 1, 19.483030929082993: 1, 29.765617819985767: 1, 28.97464959760863: 1, 30.556586042362902: 1, 31.34755426474004: 1}) \n",
      "\n",
      "number_emergency\n",
      "Counter({-0.21415691268690235: 72264, 0.8627789241060262: 6158, 1.939714760898955: 1657, 3.016650597691883: 582, 4.093586434484812: 300, 5.170522271277741: 147, 6.247458108070669: 72, 7.324393944863598: 61, 8.401329781656527: 40, 10.555201455242383: 28, 9.478265618449454: 27, 11.632137292035312: 19, 13.78600896562117: 10, 12.70907312882824: 8, 23.478431496757523: 5, 19.17068814958581: 4, 20.247623986378738: 4, 15.939880639207026: 3, 17.016816475999953: 3, 21.324559823171665: 3, 14.862944802414097: 3, 22.401495659964596: 2, 26.709239007136308: 2, 45.017148232616094: 1, 81.63296668357567: 1, 25.63230317034338: 1, 29.940046517515096: 1, 68.70973664206052: 1, 31.016982354308023: 1, 39.63246904865145: 1, 57.940378274131234: 1}) \n",
      "\n",
      "number_inpatient\n",
      "Counter({-0.5022683705624524: 54100, 0.2872903698308675: 15646, 1.0768491102241873: 6047, 1.8664078506175072: 2691, 2.655966591010827: 1285, 3.445525331404147: 661, 4.235084071797467: 388, 5.024642812190787: 220, 5.814201552584107: 119, 6.603760292977426: 94, 7.393319033370746: 50, 8.182877773764066: 40, 8.972436514157385: 29, 9.761995254550706: 16, 10.551553994944026: 9, 11.341112735337346: 7, 12.130671475730665: 4, 14.499347696910625: 2, 12.920230216123985: 1, 13.709788956517306: 1}) \n",
      "\n",
      "number_diagnoses\n",
      "Counter({0.8159869476590422: 39570, -1.2521953205380214: 9119, 0.2989413806097763: 8494, -0.2181041864394896: 8301, -0.7351497534887556: 8121, -1.7692408875872874: 4446, -2.2862864546365533: 2280, -2.803332021685819: 818, -3.320377588735085: 168, 4.4353059170039035: 37, 1.3330325147083082: 14, 2.884169215856106: 13, 2.36712364880684: 8, 1.850078081757574: 8, 3.4012147829053716: 7, 3.918260349954638: 6}) \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for col in X_train.columns:\n",
    "    print(col)\n",
    "    print(Counter(X_train[col]), '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = df_train['label'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cross Validation Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['memory', 'steps', 'verbose', 'sampling', 'model', 'sampling__k_neighbors', 'sampling__n_jobs', 'sampling__random_state', 'sampling__sampling_strategy', 'model__activation', 'model__alpha', 'model__batch_size', 'model__beta_1', 'model__beta_2', 'model__early_stopping', 'model__epsilon', 'model__hidden_layer_sizes', 'model__learning_rate', 'model__learning_rate_init', 'model__max_fun', 'model__max_iter', 'model__momentum', 'model__n_iter_no_change', 'model__nesterovs_momentum', 'model__power_t', 'model__random_state', 'model__shuffle', 'model__solver', 'model__tol', 'model__validation_fraction', 'model__verbose', 'model__warm_start'])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe = Pipeline([\n",
    "        ('sampling', SMOTE(random_state=random_state)),\n",
    "        ('model', MLPClassifier(random_state=random_state))\n",
    "    ])\n",
    "\n",
    "pipe.get_params().keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.metrics import precision_recall_curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid={\n",
    "'model__learning_rate': [\"constant\"],\n",
    "'model__hidden_layer_sizes': [(30, 10), (20, 20, 20), (20, 20, 15)],\n",
    "'model__alpha': [.015], #between .01 and .5 is best. .1 seemed best after trying .01 to 1, started getting worse at .5 \n",
    "'model__activation': [\"relu\"]\n",
    "}\n",
    "\n",
    "# Recall = TP/TP+FN -> Of the ones that are positive, how many did we indicate as positive\n",
    "# AUROC = area under curve TPR (TP/TP+FN) vs. FPR (FP/TN+FP) at various thresholds\n",
    "scoring = ['balanced_accuracy', 'accuracy', 'roc_auc', 'recall', 'precision']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = GridSearchCV(pipe, param_grid, cv=3, verbose=5, scoring=scoring, refit='roc_auc', return_train_score=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 3 candidates, totalling 9 fits\n",
      "[CV] model__activation=relu, model__alpha=0.015, model__hidden_layer_sizes=(30, 10), model__learning_rate=constant \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__activation=relu, model__alpha=0.015, model__hidden_layer_sizes=(30, 10), model__learning_rate=constant, accuracy=(train=0.836, test=0.802), balanced_accuracy=(train=0.625, test=0.543), precision=(train=0.300, test=0.175), recall=(train=0.353, test=0.209), roc_auc=(train=0.738, test=0.587), total= 2.8min\n",
      "[CV] model__activation=relu, model__alpha=0.015, model__hidden_layer_sizes=(30, 10), model__learning_rate=constant \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  2.8min remaining:    0.0s\n",
      "/Users/kalyaniasthana/opt/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/_multilayer_perceptron.py:585: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__activation=relu, model__alpha=0.015, model__hidden_layer_sizes=(30, 10), model__learning_rate=constant, accuracy=(train=0.858, test=0.828), balanced_accuracy=(train=0.589, test=0.522), precision=(train=0.320, test=0.161), recall=(train=0.244, test=0.129), roc_auc=(train=0.734, test=0.565), total= 2.9min\n",
      "[CV] model__activation=relu, model__alpha=0.015, model__hidden_layer_sizes=(30, 10), model__learning_rate=constant \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:  5.8min remaining:    0.0s\n",
      "/Users/kalyaniasthana/opt/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/_multilayer_perceptron.py:585: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__activation=relu, model__alpha=0.015, model__hidden_layer_sizes=(30, 10), model__learning_rate=constant, accuracy=(train=0.860, test=0.823), balanced_accuracy=(train=0.602, test=0.529), precision=(train=0.338, test=0.169), recall=(train=0.270, test=0.150), roc_auc=(train=0.740, test=0.575), total= 2.8min\n",
      "[CV] model__activation=relu, model__alpha=0.015, model__hidden_layer_sizes=(20, 20, 20), model__learning_rate=constant \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:  8.6min remaining:    0.0s\n",
      "/Users/kalyaniasthana/opt/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/_multilayer_perceptron.py:585: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__activation=relu, model__alpha=0.015, model__hidden_layer_sizes=(20, 20, 20), model__learning_rate=constant, accuracy=(train=0.853, test=0.833), balanced_accuracy=(train=0.583, test=0.536), precision=(train=0.299, test=0.190), recall=(train=0.235, test=0.154), roc_auc=(train=0.709, test=0.590), total= 3.3min\n",
      "[CV] model__activation=relu, model__alpha=0.015, model__hidden_layer_sizes=(20, 20, 20), model__learning_rate=constant \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed: 11.9min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__activation=relu, model__alpha=0.015, model__hidden_layer_sizes=(20, 20, 20), model__learning_rate=constant, accuracy=(train=0.830, test=0.803), balanced_accuracy=(train=0.599, test=0.539), precision=(train=0.267, test=0.171), recall=(train=0.301, test=0.200), roc_auc=(train=0.704, test=0.577), total= 2.5min\n",
      "[CV] model__activation=relu, model__alpha=0.015, model__hidden_layer_sizes=(20, 20, 20), model__learning_rate=constant \n",
      "[CV]  model__activation=relu, model__alpha=0.015, model__hidden_layer_sizes=(20, 20, 20), model__learning_rate=constant, accuracy=(train=0.851, test=0.828), balanced_accuracy=(train=0.582, test=0.534), precision=(train=0.290, test=0.183), recall=(train=0.236, test=0.155), roc_auc=(train=0.708, test=0.583), total= 2.1min\n",
      "[CV] model__activation=relu, model__alpha=0.015, model__hidden_layer_sizes=(20, 20, 15), model__learning_rate=constant \n",
      "[CV]  model__activation=relu, model__alpha=0.015, model__hidden_layer_sizes=(20, 20, 15), model__learning_rate=constant, accuracy=(train=0.869, test=0.852), balanced_accuracy=(train=0.565, test=0.531), precision=(train=0.331, test=0.208), recall=(train=0.175, test=0.118), roc_auc=(train=0.716, test=0.596), total= 2.5min\n",
      "[CV] model__activation=relu, model__alpha=0.015, model__hidden_layer_sizes=(20, 20, 15), model__learning_rate=constant \n",
      "[CV]  model__activation=relu, model__alpha=0.015, model__hidden_layer_sizes=(20, 20, 15), model__learning_rate=constant, accuracy=(train=0.842, test=0.817), balanced_accuracy=(train=0.599, test=0.537), precision=(train=0.289, test=0.177), recall=(train=0.286, test=0.175), roc_auc=(train=0.717, test=0.573), total= 1.5min\n",
      "[CV] model__activation=relu, model__alpha=0.015, model__hidden_layer_sizes=(20, 20, 15), model__learning_rate=constant \n",
      "[CV]  model__activation=relu, model__alpha=0.015, model__hidden_layer_sizes=(20, 20, 15), model__learning_rate=constant, accuracy=(train=0.880, test=0.861), balanced_accuracy=(train=0.559, test=0.522), precision=(train=0.394, test=0.205), recall=(train=0.146, test=0.085), roc_auc=(train=0.710, test=0.587), total= 1.9min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed: 22.5min finished\n",
      "/Users/kalyaniasthana/opt/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/_multilayer_perceptron.py:585: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    }
   ],
   "source": [
    "best_model = clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('sampling', SMOTE(random_state=0)),\n",
       "                ('model',\n",
       "                 MLPClassifier(alpha=0.015, hidden_layer_sizes=(20, 20, 15),\n",
       "                               random_state=0))])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_model.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([171.69406239, 158.16684031, 119.45026509]),\n",
       " 'std_fit_time': array([ 3.41153642, 30.44400394, 23.61567787]),\n",
       " 'mean_score_time': array([0.12947702, 0.18916742, 0.12085907]),\n",
       " 'std_score_time': array([0.0121719 , 0.0241942 , 0.00377016]),\n",
       " 'param_model__activation': masked_array(data=['relu', 'relu', 'relu'],\n",
       "              mask=[False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_model__alpha': masked_array(data=[0.015, 0.015, 0.015],\n",
       "              mask=[False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_model__hidden_layer_sizes': masked_array(data=[(30, 10), (20, 20, 20), (20, 20, 15)],\n",
       "              mask=[False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_model__learning_rate': masked_array(data=['constant', 'constant', 'constant'],\n",
       "              mask=[False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'model__activation': 'relu',\n",
       "   'model__alpha': 0.015,\n",
       "   'model__hidden_layer_sizes': (30, 10),\n",
       "   'model__learning_rate': 'constant'},\n",
       "  {'model__activation': 'relu',\n",
       "   'model__alpha': 0.015,\n",
       "   'model__hidden_layer_sizes': (20, 20, 20),\n",
       "   'model__learning_rate': 'constant'},\n",
       "  {'model__activation': 'relu',\n",
       "   'model__alpha': 0.015,\n",
       "   'model__hidden_layer_sizes': (20, 20, 15),\n",
       "   'model__learning_rate': 'constant'}],\n",
       " 'split0_test_balanced_accuracy': array([0.5427055 , 0.53591063, 0.53084796]),\n",
       " 'split1_test_balanced_accuracy': array([0.52234864, 0.53929765, 0.5365395 ]),\n",
       " 'split2_test_balanced_accuracy': array([0.52880341, 0.53402025, 0.52175737]),\n",
       " 'mean_test_balanced_accuracy': array([0.53128585, 0.53640951, 0.52971494]),\n",
       " 'std_test_balanced_accuracy': array([0.00849401, 0.00218318, 0.00608773]),\n",
       " 'rank_test_balanced_accuracy': array([2, 1, 3], dtype=int32),\n",
       " 'split0_train_balanced_accuracy': array([0.62504737, 0.58285815, 0.56547544]),\n",
       " 'split1_train_balanced_accuracy': array([0.58932347, 0.59857106, 0.5990406 ]),\n",
       " 'split2_train_balanced_accuracy': array([0.60192959, 0.58179945, 0.55877519]),\n",
       " 'mean_train_balanced_accuracy': array([0.60543348, 0.58774289, 0.57443041]),\n",
       " 'std_train_balanced_accuracy': array([0.01479318, 0.00766887, 0.0176157 ]),\n",
       " 'split0_test_accuracy': array([0.80229944, 0.83262704, 0.85164167]),\n",
       " 'split1_test_accuracy': array([0.82779968, 0.80266794, 0.81729742]),\n",
       " 'split2_test_accuracy': array([0.82333432, 0.82849351, 0.86143868]),\n",
       " 'mean_test_accuracy': array([0.81781115, 0.82126283, 0.84345926]),\n",
       " 'std_test_accuracy': array([0.01111889, 0.01325642, 0.01892664]),\n",
       " 'rank_test_accuracy': array([3, 2, 1], dtype=int32),\n",
       " 'split0_train_accuracy': array([0.83619848, 0.8533525 , 0.86859028]),\n",
       " 'split1_train_accuracy': array([0.85790356, 0.83000755, 0.84202089]),\n",
       " 'split2_train_accuracy': array([0.85974868, 0.85057302, 0.87981354]),\n",
       " 'mean_train_accuracy': array([0.85128357, 0.84464436, 0.8634749 ]),\n",
       " 'std_train_accuracy': array([0.01069334, 0.0104118 , 0.01584711]),\n",
       " 'split0_test_roc_auc': array([0.58690152, 0.59040345, 0.59554428]),\n",
       " 'split1_test_roc_auc': array([0.56512524, 0.57660676, 0.57345301]),\n",
       " 'split2_test_roc_auc': array([0.57452763, 0.58302861, 0.58660126]),\n",
       " 'mean_test_roc_auc': array([0.57551813, 0.58334627, 0.58519952]),\n",
       " 'std_test_roc_auc': array([0.00891768, 0.00563695, 0.00907302]),\n",
       " 'rank_test_roc_auc': array([3, 2, 1], dtype=int32),\n",
       " 'split0_train_roc_auc': array([0.73797584, 0.70927385, 0.7164363 ]),\n",
       " 'split1_train_roc_auc': array([0.73437716, 0.70360292, 0.71720018]),\n",
       " 'split2_train_roc_auc': array([0.74023483, 0.70764916, 0.71049199]),\n",
       " 'mean_train_roc_auc': array([0.73752928, 0.70684198, 0.71470949]),\n",
       " 'std_train_roc_auc': array([0.00241214, 0.00238447, 0.00299848]),\n",
       " 'split0_test_recall': array([0.20866402, 0.15410053, 0.11805556]),\n",
       " 'split1_test_recall': array([0.12929894, 0.20039683, 0.17526455]),\n",
       " 'split2_test_recall': array([0.14980159, 0.15509259, 0.08465608]),\n",
       " 'mean_test_recall': array([0.16258818, 0.16986332, 0.12599206]),\n",
       " 'std_test_recall': array([0.03363854, 0.02159425, 0.03741403]),\n",
       " 'rank_test_recall': array([2, 1, 3], dtype=int32),\n",
       " 'split0_train_recall': array([0.35333995, 0.23478836, 0.17542989]),\n",
       " 'split1_train_recall': array([0.24371693, 0.30076058, 0.28637566]),\n",
       " 'split2_train_recall': array([0.27017196, 0.23594577, 0.14566799]),\n",
       " 'mean_train_recall': array([0.28907628, 0.2571649 , 0.20249118]),\n",
       " 'std_train_recall': array([0.04670711, 0.03083042, 0.06054694]),\n",
       " 'split0_test_precision': array([0.17513183, 0.19020408, 0.20804196]),\n",
       " 'split1_test_precision': array([0.16083916, 0.17104149, 0.17702071]),\n",
       " 'split2_test_precision': array([0.16928251, 0.1826324 , 0.20512821]),\n",
       " 'mean_test_precision': array([0.16841784, 0.18129266, 0.19673029]),\n",
       " 'std_test_precision': array([0.00586691, 0.00788025, 0.01398745]),\n",
       " 'rank_test_precision': array([3, 2, 1], dtype=int32),\n",
       " 'split0_train_precision': array([0.30030916, 0.29888445, 0.33094198]),\n",
       " 'split1_train_precision': array([0.31960104, 0.26687207, 0.28914858]),\n",
       " 'split2_train_precision': array([0.33816225, 0.29027665, 0.39383102]),\n",
       " 'mean_train_precision': array([0.31935748, 0.28534439, 0.33797386]),\n",
       " 'std_train_precision': array([0.01545442, 0.01352636, 0.04302471])}"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_model.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = pd.read_csv('dataset_diabetes/diabetic_data_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = df_test[[col for col in df_test.columns if col not in remove_cols]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test['label'] = df_test.apply(lambda x: binarize_readmitted(x['readmitted']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('NO', 0.5380760538469097),\n",
       " ('>30', 0.34966100029478236),\n",
       " ('<30', 0.11226294585830796)]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(key, val/len(df_test.readmitted)) for (key, val) in Counter(df_test.readmitted).most_common()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 0.887737054141692), (1, 0.11226294585830796)]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(key, val/len(df_test.readmitted)) for (key, val) in Counter(df_test['label']).most_common()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test['age_processed'] = df_test.apply(lambda x: map_age(x['age']), axis=1)\n",
    "df_test['admission_type_processed'] = df_test.apply(lambda x: map_admission(x['admission_type_id']), axis=1)\n",
    "df_test['discharged_processed'] = df_test.apply(lambda x: discharged_home(x['discharge_disposition_id']), axis=1)\n",
    "df_test['source_processed'] = df_test.apply(lambda x: map_source(x['admission_source_id']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test['diag1_processed'] = df_test.apply(lambda x: map_diag(x['diag_1']), axis=1)\n",
    "df_test['diag2_processed'] = df_test.apply(lambda x: map_diag(x['diag_2']), axis=1)\n",
    "df_test['diag3_processed'] = df_test.apply(lambda x: map_diag(x['diag_3']), axis=1)\n",
    "df_test['change_processed'] = df_test.apply(lambda x: binarize_yn(x['change']), axis=1)\n",
    "df_test['diabetesMed_processed'] = df_test.apply(lambda x: binarize_yn(x['diabetesMed']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20353\n"
     ]
    }
   ],
   "source": [
    "df_test = df_test[df_test.gender != 'Unknown/Invalid']\n",
    "print(len(df_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test_sub = df_test[ohe_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ohe = ohe.fit_transform(df_test_sub).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# obtain column names for one hot encoded columns \n",
    "feature_labels = []\n",
    "feature_labels_ = ohe.categories_\n",
    "for col, values in zip(df_test_sub.columns, feature_labels_):\n",
    "    for val in values:\n",
    "        feature_labels.append(col+'_'+val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test_ohe = pd.DataFrame(test_ohe, columns = feature_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test_num = df_test[num_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test_num_scaled = scaler.fit_transform(df_test_num.astype(np.float64))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test_num = pd.DataFrame(df_test_num_scaled, index=df_test_num.index, columns=df_test_num.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20353, 87) (20353, 12)\n"
     ]
    }
   ],
   "source": [
    "print(df_test_ohe.shape, df_test_num.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(81410, 87) (81410, 12)\n"
     ]
    }
   ],
   "source": [
    "print(df_train_ohe.shape, df_train_num.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20353, 41)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(81410, 41)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_clf = best_model.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score,balanced_accuracy_score\n",
    "from sklearn.metrics import precision_recall_curve, recall_score, precision_score\n",
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_test = pd.concat([df_test_ohe, df_test_num], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_test = df_test['label'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test_ohe.reset_index(drop=True, inplace=True)\n",
    "df_test_num.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = pd.concat([df_test_ohe, df_test_num], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test = df_test['label'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_pred = final_clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'precisions' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-67-eb227a2fea1e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfigsize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m6\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Precision vs Recall'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0mplot_precision_vs_recall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprecisions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrecalls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;31m# plt.plot([recall_90_precision, recall_90_precision], [0., 0.9], \"r:\")\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;31m# plt.plot([0.0, recall_90_precision], [0.9, 0.9], \"r:\")\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'precisions' is not defined"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAesAAAF1CAYAAADIn8KSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAVeklEQVR4nO3df7Dld13f8debhAQLCdHuMoX8RJKAS8YKXgOOtkCJNsnoxulEm3QoxUbWgcYWQSuiAwwqVpAyOKZCBEQYSQh0hB0mNFoaCiJhskyUYRMzrjGQJdAskEAohBB894/zXXq4ubv37Obe3c/ufTxm7sw53/O553zOJzf3ud/vOed7q7sDAIzrYYd7AgDA/ok1AAxOrAFgcGINAIMTawAYnFgDwODEGg6jqtpZVc9cZcxpVfXVqjrmEE3riFBVXVVnTpffVlW/ebjnBOtFrGEFVXV7VX19iuT/qao/qqpHrfXjdPeTu/tDq4z5THc/qru/tdaPvxamUN4/rdWXqurPq+pJh3tecDQRa9i3n+zuRyV5apIfSvLrywfUjP+PktdMa3Vyks8mecthng8cVfySgVV092eTfCDJOUlSVR+qqt+qqo8m+VqS762qR1fVW6rqc1X12ar6zfnD1lX1/Kq6paruraqbq+qp0/bbq+q86fK5VbWjqr4y7c3/12n7GdMh32On64+rqu3TXuyuqnr+3OO8sqquqaq3T4+1s6qWVnpeVfXGqvrdZdveV1Uvni7/yvRc7q2qW6vq2Qus1deTXJPkB5bd77+fnv/dVXVdVZ0+d9uTp73xL03P+2Vz6/GxqrpnWtffr6rjVpsDHI3EGlZRVacmuTDJTXOb/22SbUlOSPLpJH+c5IEkZyZ5SpIfT/Jz0/f/dJJXJnlukhOTbE3yxRUe6g1J3tDdJyZ5QmbRW8lVSXYneVySi5O8ellItya5OslJSbYn+f193M87k/zrqqppnt89zfvqqnpiksuT/FB3n5DkXya5fR/3821V9cgklybZNbftp5K8LMm/SrI5yUem55CqOiHJ/0zyP6bnc2aSD07f+q0kv5hkU5IfTvLsJC9cbQ5wNBJr2Lf3VtU9Sf4iyf9O8uq5297W3Tu7+4Ek35PkgiQv6u7/2913JXl9kkumsT+X2WHiG3tmV3d/eoXH+2aSM6tqU3d/tbtvWD5g+ofDjyb5le6+r7v/KsmbM/vHw15/0d3XTq9xvyPJP93H8/tIkk7yz6brFyf5WHffmVkoj0+ypaoe3t23d/ff7Wetfmlaq3un+c3P5+eT/HZ33zKt16uT/MC0d/0TST7f3a+bns+93f3xJOnuT3T3Dd39QHffnuRNSZ6xnznAUUusYd9+qrtP6u7Tu/uF0yHeve6Yu3x6kocn+dx0yPaezMLymOn2U5PsL3R7XZbk7CR/U1U3VtVPrDDmcUm+1N33zm37dGavFe/1+bnLX0vyiL2H0Of17K/4XJ3ZnnCS/JskfzLdtivJizI7InBXVV1dVY/bz9x/t7tPSnJGkq8neeLcbacnecPc2nwpSU1z3ufaVNXZVfX+qvp8VX0ls8hv2s8c4Kgl1nBw5v9c3R1JvpFk0xT3k7r7xO5+8tztT1j1Drv/trsvzSzyv5PkPdNh5Xl3Jvme6fDxXqdl9qaug3FVkounvdynJfnvc/N5Z3f/aGax7WlOqz2HzyT5T5nF+bumzXck+fm5tTmpu7+ru/8y+1+bP0jyN0nOml4aeFlmkYcNR6zhIeruzyX5sySvq6oTq+phVfWEqtp7yPbNmR0m/sHp3eNnzr/Baq+qek5Vbe7uf0hyz7T5Oz6u1d13JPnLJL9dVY+oqu/PbI/8Tw5y7jcl2TPN8bruvmeayxOr6l9U1fFJ7stsb3mhj451959n9o+KbdOmNyb51ap68nTfj55ex0+S9yf5J1X1oqo6vqpOqKqnTbedkOQrSb46fRTsBQfzHOFoINawNp6b5LgkNye5O8l7kjw2Sbr73Ul+K7M3dN2b5L2Zvc693PlJdlbVVzN7s9kl3X3fCuMuzexw851J/jTJK6ZAHqyrkpw3zW+v45P8lyRfyOyw+mMy27Nd1GuT/OeqOr67/zSzvfKrp8PZn8rsNf5Mh/N/LMlPTo/zt0meNd3HL2V2aP7eJH+Y5F0H8+TgaFCzl60AgFHZswaAwa0a66p6a1XdVVWf2sftVVW/N52c4ZN7T/YAAKyNRfas35bZa2n7ckGSs6avbZm9gxMAWCOrxrq7P5zZ5yL35aIkb59O9nBDkpOq6rFrNUEA2OjW4jXrk/OdJ4jYne88QQMA8BA86KxGB2GlkxSs+BbzqtqW6bOXj3zkI3/wSU/yV/QA2Bg+8YlPfKG7Nx/M965FrHdndsrAvU7J7POfD9LdVya5MkmWlpZ6x44da/DwADC+qlrpbwIsZC0Og29P8tzpXeFPT/Ll6YxOAMAaWHXPuqquSvLMJJuqaneSV2T2RwvS3W9Mcm1mfz5wV2Z/NOBn12uyALARrRrr6Q8L7O/2TvIf1mxGAMB3cAYzABicWAPA4MQaAAYn1gAwOLEGgMGJNQAMTqwBYHBiDQCDE2sAGJxYA8DgxBoABifWADA4sQaAwYk1AAxOrAFgcGINAIMTawAYnFgDwODEGgAGJ9YAMDixBoDBiTUADE6sAWBwYg0AgxNrABicWAPA4MQaAAYn1gAwOLEGgMGJNQAMTqwBYHBiDQCDE2sAGJxYA8DgxBoABifWADA4sQaAwYk1AAxOrAFgcGINAIMTawAYnFgDwODEGgAGJ9YAMDixBoDBiTUADE6sAWBwYg0AgxNrABicWAPA4MQaAAYn1gAwOLEGgMGJNQAMTqwBYHBiDQCDWyjWVXV+Vd1aVbuq6qUr3H5aVV1fVTdV1Ser6sK1nyoAbEyrxrqqjklyRZILkmxJcmlVbVk27NeTXNPdT0lySZL/ttYTBYCNapE963OT7Oru27r7/iRXJ7lo2ZhOcuJ0+dFJ7ly7KQLAxnbsAmNOTnLH3PXdSZ62bMwrk/xZVf1CkkcmOW9NZgcALLRnXSts62XXL03ytu4+JcmFSd5RVQ+676raVlU7qmrHnj17Dny2ALABLRLr3UlOnbt+Sh58mPuyJNckSXd/LMkjkmxafkfdfWV3L3X30ubNmw9uxgCwwSwS6xuTnFVVj6+q4zJ7A9n2ZWM+k+TZSVJV35dZrO06A8AaWDXW3f1AksuTXJfklsze9b2zql5VVVunYS9J8vyq+uskVyV5XncvP1QOAByERd5glu6+Nsm1y7a9fO7yzUl+ZG2nBgAkzmAGAMMTawAYnFgDwODEGgAGJ9YAMDixBoDBiTUADE6sAWBwYg0AgxNrABicWAPA4MQaAAYn1gAwOLEGgMGJNQAMTqwBYHBiDQCDE2sAGJxYA8DgxBoABifWADA4sQaAwYk1AAxOrAFgcGINAIMTawAYnFgDwODEGgAGJ9YAMDixBoDBiTUADE6sAWBwYg0AgxNrABicWAPA4MQaAAYn1gAwOLEGgMGJNQAMTqwBYHBiDQCDE2sAGJxYA8DgxBoABifWADA4sQaAwYk1AAxOrAFgcGINAIMTawAYnFgDwODEGgAGJ9YAMDixBoDBiTUADE6sAWBwYg0Ag1so1lV1flXdWlW7quql+xjzM1V1c1XtrKp3ru00AWDjOna1AVV1TJIrkvxYkt1Jbqyq7d1989yYs5L8apIf6e67q+ox6zVhANhoFtmzPjfJru6+rbvvT3J1kouWjXl+kiu6++4k6e671naaALBxLRLrk5PcMXd997Rt3tlJzq6qj1bVDVV1/kp3VFXbqmpHVe3Ys2fPwc0YADaYRWJdK2zrZdePTXJWkmcmuTTJm6vqpAd9U/eV3b3U3UubN28+0LkCwIa0SKx3Jzl17vopSe5cYcz7uvub3f33SW7NLN4AwEO0SKxvTHJWVT2+qo5LckmS7cvGvDfJs5KkqjZldlj8trWcKABsVKvGursfSHJ5kuuS3JLkmu7eWVWvqqqt07Drknyxqm5Ocn2SX+7uL67XpAFgI6nu5S8/HxpLS0u9Y8eOw/LYAHCoVdUnunvpYL7XGcwAYHBiDQCDE2sAGJxYA8DgxBoABifWADA4sQaAwYk1AAxOrAFgcGINAIMTawAYnFgDwODEGgAGJ9YAMDixBoDBiTUADE6sAWBwYg0AgxNrABicWAPA4MQaAAYn1gAwOLEGgMGJNQAMTqwBYHBiDQCDE2sAGJxYA8DgxBoABifWADA4sQaAwYk1AAxOrAFgcGINAIMTawAYnFgDwODEGgAGJ9YAMDixBoDBiTUADE6sAWBwYg0AgxNrABicWAPA4MQaAAYn1gAwOLEGgMGJNQAMTqwBYHBiDQCDE2sAGJxYA8DgxBoABifWADA4sQaAwYk1AAxuoVhX1flVdWtV7aqql+5n3MVV1VW1tHZTBICNbdVYV9UxSa5IckGSLUkuraotK4w7Icl/TPLxtZ4kAGxki+xZn5tkV3ff1t33J7k6yUUrjPuNJK9Jct8azg8ANrxFYn1ykjvmru+etn1bVT0lyand/f793VFVbauqHVW1Y8+ePQc8WQDYiBaJda2wrb99Y9XDkrw+yUtWu6PuvrK7l7p7afPmzYvPEgA2sEVivTvJqXPXT0ly59z1E5Kck+RDVXV7kqcn2e5NZgCwNhaJ9Y1Jzqqqx1fVcUkuSbJ9743d/eXu3tTdZ3T3GUluSLK1u3esy4wBYINZNdbd/UCSy5Ncl+SWJNd0986qelVVbV3vCQLARnfsIoO6+9ok1y7b9vJ9jH3mQ58WALCXM5gBwODEGgAGJ9YAMDixBoDBiTUADE6sAWBwYg0AgxNrABicWAPA4MQaAAYn1gAwOLEGgMGJNQAMTqwBYHBiDQCDE2sAGJxYA8DgxBoABifWADA4sQaAwYk1AAxOrAFgcGINAIMTawAYnFgDwODEGgAGJ9YAMDixBoDBiTUADE6sAWBwYg0AgxNrABicWAPA4MQaAAYn1gAwOLEGgMGJNQAMTqwBYHBiDQCDE2sAGJxYA8DgxBoABifWADA4sQaAwYk1AAxOrAFgcGINAIMTawAYnFgDwODEGgAGJ9YAMDixBoDBiTUADE6sAWBwYg0AgxNrABjcQrGuqvOr6taq2lVVL13h9hdX1c1V9cmq+mBVnb72UwWAjWnVWFfVMUmuSHJBki1JLq2qLcuG3ZRkqbu/P8l7krxmrScKABvVInvW5ybZ1d23dff9Sa5OctH8gO6+vru/Nl29IckpaztNANi4Fon1yUnumLu+e9q2L5cl+cBDmRQA8P8du8CYWmFbrziw6jlJlpI8Yx+3b0uyLUlOO+20BacIABvbInvWu5OcOnf9lCR3Lh9UVecl+bUkW7v7GyvdUXdf2d1L3b20efPmg5kvAGw4i8T6xiRnVdXjq+q4JJck2T4/oKqekuRNmYX6rrWfJgBsXKvGursfSHJ5kuuS3JLkmu7eWVWvqqqt07DXJnlUkndX1V9V1fZ93B0AcIAWec063X1tkmuXbXv53OXz1nheAMDEGcwAYHBiDQCDE2sAGJxYA8DgxBoABifWADA4sQaAwYk1AAxOrAFgcGINAIMTawAYnFgDwODEGgAGJ9YAMDixBoDBiTUADE6sAWBwYg0AgxNrABicWAPA4MQaAAYn1gAwOLEGgMGJNQAMTqwBYHBiDQCDE2sAGJxYA8DgxBoABifWADA4sQaAwYk1AAxOrAFgcGINAIMTawAYnFgDwODEGgAGJ9YAMDixBoDBiTUADE6sAWBwYg0AgxNrABicWAPA4MQaAAYn1gAwOLEGgMGJNQAMTqwBYHBiDQCDE2sAGJxYA8DgxBoABifWADA4sQaAwYk1AAxuoVhX1flVdWtV7aqql65w+/FV9a7p9o9X1RlrPVEA2KhWjXVVHZPkiiQXJNmS5NKq2rJs2GVJ7u7uM5O8PsnvrPVEAWCjWmTP+twku7r7tu6+P8nVSS5aNuaiJH88XX5PkmdXVa3dNAFg41ok1icnuWPu+u5p24pjuvuBJF9O8o/XYoIAsNEdu8CYlfaQ+yDGpKq2Jdk2Xf1GVX1qgcfn4G1K8oXDPYkNwDqvP2u8/qzx+nviwX7jIrHeneTUueunJLlzH2N2V9WxSR6d5EvL76i7r0xyZZJU1Y7uXjqYSbMYa3xoWOf1Z43XnzVef1W142C/d5HD4DcmOauqHl9VxyW5JMn2ZWO2J/l30+WLk/yv7n7QnjUAcOBW3bPu7geq6vIk1yU5Jslbu3tnVb0qyY7u3p7kLUneUVW7MtujvmQ9Jw0AG8kih8HT3dcmuXbZtpfPXb4vyU8f4GNfeYDjOXDW+NCwzuvPGq8/a7z+DnqNy9FqABib040CwODWPdZOVbr+FljjF1fVzVX1yar6YFWdfjjmeSRbbY3nxl1cVV1V3lV7EBZZ56r6menneWdVvfNQz/FIt8Dvi9Oq6vqqumn6nXHh4Zjnkayq3lpVd+3r48k183vTf4NPVtVTV73T7l63r8zekPZ3Sb43yXFJ/jrJlmVjXpjkjdPlS5K8az3ndLR9LbjGz0ryj6bLL7DGa7/G07gTknw4yQ1Jlg73vI+0rwV/ls9KclOS756uP+Zwz/tI+lpwja9M8oLp8pYktx/ueR9pX0n+eZKnJvnUPm6/MMkHMjtHydOTfHy1+1zvPWunKl1/q65xd1/f3V+brt6Q2WflWdwiP8dJ8htJXpPkvkM5uaPIIuv8/CRXdPfdSdLddx3iOR7pFlnjTnLidPnRefB5NVhFd384K5xrZM5FSd7eMzckOamqHru/+1zvWDtV6fpbZI3nXZbZv+hY3KprXFVPSXJqd7//UE7sKLPIz/LZSc6uqo9W1Q1Vdf4hm93RYZE1fmWS51TV7sw+BfQLh2ZqG8qB/t5e7KNbD8GanaqUfVp4/arqOUmWkjxjXWd09NnvGlfVwzL7a3PPO1QTOkot8rN8bGaHwp+Z2RGij1TVOd19zzrP7WixyBpfmuRt3f26qvrhzM6hcU53/8P6T2/DOODurfee9YGcqjT7O1Up+7TIGqeqzkvya0m2dvc3DtHcjharrfEJSc5J8qGquj2z16C2e5PZAVv098X7uvub3f33SW7NLN4sZpE1vizJNUnS3R9L8ojMzhvO2lno9/a89Y61U5Wuv1XXeDpE+6bMQu01vgO33zXu7i9396buPqO7z8jsfQFbu/ugzwO8QS3y++K9mb1hMlW1KbPD4rcd0lke2RZZ488keXaSVNX3ZRbrPYd0lke/7UmeO70r/OlJvtzdn9vfN6zrYfB2qtJ1t+AavzbJo5K8e3rv3me6e+thm/QRZsE15iFacJ2vS/LjVXVzkm8l+eXu/uLhm/WRZcE1fkmSP6yqX8zs0Ozz7EAdmKq6KrOXajZNr/2/IsnDk6S735jZewEuTLIrydeS/Oyq9+m/AQCMzRnMAGBwYg0AgxNrABicWAPA4MQaAAYn1gAwOLEGgMGJNQAM7v8BLSyLPjYBm8cAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_precision_vs_recall(precision, recalls):\n",
    "    plt.plot(recalls, precisions, \"b-\", linewidth=2)\n",
    "    plt.xlabel(\"Recall\", fontsize=16)\n",
    "    plt.ylabel(\"Precision\", fontsize=16)\n",
    "#     plt.axis([0, 1, 0, 1])\n",
    "    plt.grid(True)\n",
    "\n",
    "plt.figure(figsize=(8, 6))\n",
    "plt.title('Precision vs Recall')\n",
    "plot_precision_vs_recall(precisions, recalls)\n",
    "# plt.plot([recall_90_precision, recall_90_precision], [0., 0.9], \"r:\")\n",
    "# plt.plot([0.0, recall_90_precision], [0.9, 0.9], \"r:\")\n",
    "# plt.plot([recall_90_precision], [0.9], \"ro\")\n",
    "# save_fig(\"precision_vs_recall_plot\")\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpr, tpr, _ = roc_curve(y_test, y_test_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf4AAAGICAYAAACgFIL5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdd3RU1d7G8e9OpyV0gdAFERASAdFguQiiICJNIGC5FooFNAFEUeyiKE1RlKK+6hUxgHQpNlCKUoSE3nuHBAIhpM5+/5igCKQMJJmU57MWy5lz9jnzZJbwyz5nn72NtRYREREpHDzcHUBERERyjwq/iIhIIaLCLyIiUoio8IuIiBQiKvwiIiKFiAq/iIhIIaLCLyIiUoio8IsUQsaYxcaYk8YY34u29byoXXNjzIEL3ltjzFljTJwx5qAxZpQxxvOiY+4zxqxMaxdtjJlkjKl8UZuKxpjPjTGHjTFnjDFbjDFvGGOK5dTPLCJOKvwihYwxpjpwO2CB+6/gFEHW2uLAf4BuwOMXnPsB4FvgQ6AsUB9IBJYaY0qltSkN/AEUAUKstSWAVkBJ4Nor+qFEJMtU+EUKn0eAP4Evgf9e6UmstTuAZUAwgDHGACOBt621k6y156y1R4CeQBwQnnZof+AM8JC1dk/aufZba5+z1q670jwikjUq/CKFzyPApLQ/9xhjrrmSkxhjrsd55WBH2qY6QFVg6oXtrLUO4HucvXqAu4DpadtFJJep8IsUIsaY24BqwBRr7V/ATqCHi6dZY4w5C2wGFgOfpG0vm/bfw5c55vAF+8uk00ZEcoEKv0jh8l/gR2vtibT33/LP5f4UwPui9t5A8kXbGgHFcd7fvxk4PyDv/DkrXuZzK16wPzqdNiKSC1T4RQoJY0wRoCvwH2PMEWPMEZz33YOMMUHAPqD6RYfVAPZefC7rNAXnIL1X0zZvBQ4AXS76XA+gM/BL2qafgY5p20Ukl+kvnkjh0QFIBerhHJAXDNQFluC87x8BPGaMaWqcrsP5i8F3GZxzGNDbGFPBOtf4HggMMcb0MMYUMcZUAD4D/IHRaceMSnv/lTGmGoAxJjDt0cCG2fwzi8hFVPhFCo//Av9nrd1nrT1y/g/wMfAgzh75i8D/AbHAPOArYEJ6J7TWrgd+A55Pex8BPIzzF4YTwCacj+3daq2NTmsTAzTDeQthhTHmTNpnx/LPQEERySHG+Uu6iIiIFAbq8YuIiBQiuVr4jTFfGGOOGWM2pLPfGGPGGGN2GGPWGWMa5WY+ERGRgi63e/xfAq0z2N8GqJ32pzfwaS5kEhERKTRytfBba38HYjJo0h74Ou1RoT+BksYYPe8rIiKSTfLaPf5AYP8F7w+kbRMREZFs4OXuABcxl9l22ccOjDG9cd4OwM/Pr3HVqlVzMleh53A48PDIa78nFjz6nnOevuOcp+84+1kg+kwiJ48fxqYkAZyw1pa7knPltcJ/AKhywfvKwKHLNbTWTiDt+eI6derYrVu35ny6Qmzx4sU0b97c3TEKPH3POU/fcc7Td5y9dhw7w2PDI9jxYV+MTxEeeWUMX7325CUzamZVXvuVbDbwSNro/luAWGutFvMQEZFCx+GwfLF0F23HLGWfKU/FkPbM+XU5X77a56rOm9uP803GObd3HWPMAWPME8aYJ40xT6Y1mQfswjl710Tg6dzMJyIikhccjj3HvUO+4OnQ+4iPjaHrTdXYsvAb7r253lWfO1cv9Vtru2ey3wLP5FIcERGRPGfm2gM8+fyrHF30NT7+ZXmpeUWe6hyUbefPa/f4RURECqXY+GT6f7WYb98bRMLeSKrf1JIfv/+G2lUqZOvn5LV7/CIiIoXO0u0nuOeD34n45D0SD27miReHsfPPH7O96IN6/CIiIm6TkJzK0Nnr+XLxRjyLleSOh8MJv3UYrW5tkmOfqcIvIiLiBhsOxtJ77A/89eXrGOPJe1/O5JkWtfHyzNmL8Sr8IiIiuSgl1cH433fx5gcTOL7gYzw9PXnvg7E816pOrny+Cr+IiEgu2Rt9ln5f/8GvX7zH2Q0/U6Xujfw0axp1atfMtQwa3CciIpLDrLVMXrmPNh8uIXL/KVKPbuO/z/Rn17qVuVr0QT1+ERGRHHX8TCIvTIti9tRvKVr3du5vUpMhQ9ZTobS/W/Ko8IuIiOSQhRuPMOh/S9n+/XDO7VhJ56ByfNyjs1szqfCLiIhks7jEFN6cs5Gvp8/jxJwR2IQzvDlsBEMG9Xd3NBV+ERGR7LRqTwz9p0SyadEsYhZ8xDVVqvPDjJ9o3LiRu6MBKvwiIiLZIinFweiftzHut51YCw2a3ELF8nFMHPshxYsXd3e8v6nwi4iIXKWtR84QHhHJ6t8WcG7nal557wPC7qqDj1eGa9O5hQq/iIjIFXI4LF8s282wues4+tNE4tbOo27DYHrfUhEfr7z5xLwKv4iIyBU4eOocA6dE8dvKtZyY/T7Jx/fQLyycEe8Nw8fHx93x0qXCLyIi4gJrLTMjD/LqrI2cPptA9PdvUNQjhcnz5tGmTRt3x8uUCr+IiEgWnYpP4uUZG5izeifG25dWNwTSeVoEwfWuo2LFiu6OlyUq/CIiIlnw27bjPD81iv1booieO5yO3R5m4iPvY4xxdzSXqPCLiIhk4FxSKsPmb+bL5bs5veJ7Ypd8Q2DlyoQ/2jnfFX1Q4RcREUlX1P5ThE+JZNvu/cT8MJJze6Lo0qUrEyaMp2TJku6Od0VU+EVERC6Skupg7KKdjPl1O6kOSyXPM8RG72bixIk88cQT+bKnf54Kv4iIyAV2nzhLeEQka/cc59yuv+j3WHeev6c1CW/0yLe9/Aup8IuIiOB8TG/Sin0M/WEzp4/tJ/aHEZw9uI0Or3fFz9sTvwJQ9EGFX0REhGOnExj0/ToWbz1O3MZFnP75U4oV8WX69Ok0aNDA3fGylQq/iIgUags2HGbw9PWcjE8m7tfxRK+aw2233cakSZOoWrWqu+NlOxV+EREplE4nJPPG7E18v+YAALfXLsuNgfcS2+ZGXnnlFby8CmaJLJg/lYiISAb+3BXNgClRHDgZz7m1c+gYcj2fPnYvHh43uztajlPhFxGRQiMxJZVRP25jwpJdpJyNJenXsRzfuJwzZbvj4ZF/H9FzhQq/iIgUCpsPnyY8IpItR86QuG8d8T9+QMKZU4wZM4a+ffu6O16uUeEXEZECLdVh+WzJLkb+uI2kVAflU0/wV8QQateuzXc/LyQ4ONjdEXOVCr+IiBRY+2PiGTA1ipW7Y3AkJ/LQbbV5+d57mBHkSYcOHShevLi7I+Y6FX4RESlwrLVMX3OQ12ZvJC4xBc+9Kzn986d07jGfYr5ePPTQQ+6O6DYq/CIiUqDEnE3i5Rnrmb/hCI7kREpEfsumX7+nSZMmBWLK3aulwi8iIgXGoq3HGDRtHcfPJOIVe4Ckn0azaedWBg4cyNChQ/Hx8XF3RLdT4RcRkXwvPimFoT9sZtKKfQA0rV6awN3LmXQ6hvnz59O6dWs3J8w7VPhFRCRfW7vvJP2nRLH7xFk8ks/Sva4fbz5+C9ZxEy/2f5ZrrrnG3RHzFBV+ERHJl5JTHXz06w7GLtpBqsNyzbm9HJj+Hv8zltceaY2Pj4+K/mWo8IuISL6z83gc4RGRrDsQCzioc+RXfvnmI6pUqcJ3332ne/kZUOEXEZF8w1rL//7cyzvzNpOQ7KBCEYvjp5H8uOw3unXrxvjx4wkICHB3zDxNhV9ERPKFo6cTeH7aOn7fdhyATo0Cea1dPXpv+h99PvuMxx9/HGMKx3z7V0OFX0RE8rwf1h3mpRnriT2XjL8P1D20kGeb9iegiA8RERHujpeveLg7gIiISHpizyUTHhHJM9+uIfZcMjcGJOAx51WmfP4Rs2fPdne8fEk9fhERyZOW7zzBwClRHIpNwM/bg+aeW5n0zit4e3szY8YMOnTo4O6I+ZIKv4iI5CkJyakMX7iVz5fuBiCocgAhKVEMDg/n9ttvZ9KkSVSpUsXNKfMvFX4REckzNh6KJTwikm1H4/D0MDzznxr0u6sOieeC8SOZvn374uWl0nU1dI9fRETcLtVh+XTxTjqMXca2o3HUKFOUzj7rmPTSQyQnJlC8eHHCwsJU9LOBCr+IiLjVvuh4Qif8wXsLtpCcaulcrwR+i0Yw/I3BBAYGkpiY6O6IBYp+dRIREbew1jJ19QHemLORs0mplC/hS/fKpxn+Yi+io6MZM2YMffv21bP52UyFX0REct2JuEQGT1/PT5uOAnBvgwq83f4G2rS8A39/f+bNm0dwcLCbUxZMKvwiIpKrft50lBenr+NEXBIlfL3o2zSALiE1KF3cl++//55SpUpRrFgxd8cssFT4RUQkV5xNTOHtHzYxeeV+AG6pWZoWfnt44aFQlrVrx9dff03lypXdnLLg0+A+ERHJcX/tjaHNh0uYvHI/Pp4eDLqrBgF/fUmf//agVq1avPbaa+6OWGioxy8iIjkmKcXBmF+288niHTgs1K3oT9hNxRnYpxsbNmxg4MCBDB06VMvo5iIVfhERyRHbj54hfEokGw6exhh48j/XEt6qNjHHjwEwf/58Wrdu7eaUhY/Lhd8YUwqoCBQBTgAHrLWp2R1MRETyJ4e1/N+y3Qybv4XEFAeVSxXhjdY1+GPOJLzuHkzFihWJiorCw0N3m90hS4XfGHMD0BNoDdS+aHe8MeYPYDLwnbX2XPZGFBGR/OJw7DlGrk5gY/QmALo2qczdZU/zRPs7OXjwIC1btiQkJERF340y/OaNMQ2NMQuAdTiL/hKgHxAKtAceA0YAKcDHwAFjzABjjHcG52xtjNlqjNlhjHnxMvurGmMWGWPWGmPWGWPuveKfTkREcs3sqEPcM/p3NkY7KF3Mh7Hdgym57Qda33UnHh4eLF26lJCQEHfHLPQy6/H/CXwFNLbWrs2ooTGmONAZGAj4AUMv08YTGAu0Ag4Aq4wxs621my5oNgSYYq391BhTD5gHVM/ajyMiIrktNj6ZIbM2MCfqEABB5TyZ2Pt2Xnzuaf7v//6P0NBQxo0bR0BAgJuTCmRe+OtYa/dn5UTW2jjgK2PM10CldJo1BXZYa3cBGGO+w3nl4MLCbwH/tNcBwKGsfL6IiOS+pdtPMHBqFEdOJ1DUx5MhbetR4ewOypfwo1evXtx222089thjmnY3D8mw8Ge16F90jAUOprM7ELjwnAeAmy9q8zrwozGmH1AMuMvVDCIikrMSklMZNn8LXy7fA8CNVUsyrH1dxo98m/3799OiRQtCQkJ0aT8Pyu3H+S73K5+96H134Etr7UhjTAjwP2PMDdZax79OZExvoDdAuXLlWLx4cU7klTRxcXH6jnOBvuecp+/46u2JTWXCukQOnbV4Gmhfy5sg7120u/Mxtm/fzn333ceiRYvUy8+jMiz8xpjNXFqY02OttfUzaXMAqHLB+8pcein/CZwDCbHW/mGM8QPKAscu+rAJwASAOnXq2ObNm2cxplyJxYsXo+845+l7znn6jq9cSqqDcb/t5IMV20lxWK4tV4wPut1I5KLZPP3003h7ezNjxgxKliyp7zgPy6zHH0XWC39WrAJqG2Nq4LwdEAr0uKjNPqAl8KUxpi7OgYLHszGDiIi4aG/0WcIjIlmz7xQAjzarzottrufE0cP06dOHJk2aMGnSJKpUqaIrKnlcZvf4Q7Pzw6y1KcaYvsBCwBP4wlq70RjzJrDaWjsbGABMNMaE4/yl49G0cQMiIpLLrLV8t2o/b83dRHxSKtf4+zKiSxCVveLw8/akcuXK/P777wQHB+Plpclg84Ncn0HBWjvPWnudtfZaa+3QtG2vphV9rLWbrLW3WmuDrLXB1tofczujiIjA8TOJ9PxqNYOnryc+KZX7GlZkwXO3s3ruN9SpU4fJkycD0KRJExX9fCSze/xdXTmZtXbK1cUREZG8YOHGIwyevp6Ys0n4+3nxVocbaBbow0NdOzFv3jzatWvH3Xff7e6YcgUy+xXtOxfOZQEVfhGRfCwuMYU352xkyuoDANxaqwzDHwhiW+QKgu7tQXR0NGPGjKFv374atZ9PZVb46+ZKChERcbtVe2LoPyWS/THn8PHy4MXW1/Nos+p4eBj+jInB39+fefPmERwc7O6ochUyG9y3NbeCiIiIeySlOBj98zbG/bYTa6F+JX8+6BaMT0IM06ZNpWvXrnTq1In77rsPHx8fd8eVq6TRGCIihdjWI2cIi4hk8+HTeBh45s5aPNuyNnNmzaBnz554eXnRpk0bSpQooaJfQLhU+I0x/wGeBOrgfL7+QlmZwEdERPIAh8PyxbLdvL9wK0kpDqqWLsqorkHUv6YIz/Z9mvHjx9O0aVMmT55MiRIl3B1XslGWC78xphUwH1gGBAO/AsWBJsBeYEVOBBQRkex18NQ5Bk6J4o9d0QCE3lSFIffVw5tUmjZtyrp16xg0aBBvvfWWevkFkCs9/teAiUBfIBkYZK1dY4y5AfgBmJoD+UREJJtYa5kZeZBXZ27kTGIKZYv78G6nhrSqd01aCy8eeeQRGjRooEf1CjBXJvCpB8wAzi+W4wVgrd2Ac0W917MzmIiIZJ9T8Un0/XYt4RFRnElM4a6617Ag7A6aVPShW7du/PLLLwAMGDBARb+Ac3XmvqS06XOP41xg57z9QO1sSyUiItnmt23HuXv07/yw/jDFfDx5v3NDJj7SmG3r/iI4OJjp06ezY8cOd8eUXOLKpf6tQLW012uAfsaYRUAq8BzOxXVERCSPOJeUyrvzN/P1H3sBaFKtFKO6BhNY0pd3332XV199lWrVqrFs2TKaNm3q5rSSW1wp/BFAUNrr14GfgBNp7y3wSPbFEhGRqxG1/xThEZHsOnEWb09DeKvr6HPHtXh6GKZOncrLL79MaGgo48aNIyAgwN1xJRdlufBbaz+44PUKY0wQcB9QBPjRWhuZA/lERMQFKakOxi7ayZhft5PqsNQuX5zR3YK5ITCA48ePU65cOR544AHmz5/PPffco2l3C6ErnsDHWrsb+Cgbs4iIyFXYdTyO/lOiiNx/CoAnbqvB8/fUwThSCAsL4+uvvyYyMpKqVavSunVrN6cVd3HlOf57gGrW2gmX2dcb2G2t/Sk7w4mISOastUxasY+hP2zmXHIqFQP8GNEliFtrlWXbtm2Ehoaydu1a+vbtS/ny5d0dV9zM1ef456Szr2TafhV+EZFcdOx0AoO+X8firccB6BBciTfa30BAEW/+97//8dRTT+Hr68vMmTNp3769m9NKXuBK4a9P+s/qrwUGX3UaERHJsvnrD/PSjPWcjE8moIg3b3e4gXZBlf7e//PPP9O4cWO++eYbqlSp4sakkpe4Uvg9cQ7ku5yigOZ1FBHJBacTknl99kamrzkIwO21yzL8gSAqBPjx119/4efnR/369Rk3bhze3t54eWk9NvmHKxP4rAdC09nXDdhw9XFERCQjf+6Kps0HS5i+5iC+Xh682b4+Xz/elPIlfBg1ahQhISEMGDAAgCJFiqjoyyVc+T9iNPCdMSYF55z9B4BAoDfOwt8j++OJiAhAYkoqI3/cxsQlu7AWGlYOYFTXYGqVL86xY8d47LHHmDdvHu3bt+fzzz93d1zJw1x5jn+KMaYq8Cb/LvIJOBfsicjucCIiApsPnyY8IpItR87g6WF4pkUt+rWohbenB1u2bKFFixbExMTw8ccf8/TTT+vZfMmQS9eArLUjjDFfAHcApXHO3LfEWnsyJ8KJiBRmqQ7LZ0t2MfLHbSSlOqhepiijugXTqGqpv9vUrFmTO++8k0GDBhEUFJTB2UScXL75Y62NAWbmQBYREUmzPyaeAVOjWLk7BoAHb67Ky23rUtTHiz179vDCCy8wbtw4SpUqxaRJk9ycVvITl1bnM8ZcY4x5xxiz1BizyRhTL23708aYJjkTUUSk8LDWMu2vA7T5cAkrd8dQtrgvXzzahKEdG1DUx4tp06YRHBzMggUL2LBBY6rFda7M3Hc98DvgDawCQgC/tN11gGbAQ9kdUESksIg5m8RL09ezYOMRAO6pfw3vdGxAmeK+xMfHEx4ezoQJE2jatCmTJ0+mZs2abk4s+ZErl/pHALuBe4A4IOmCfcuAd7Mxl4hIobJoyzGen7aOE3GJFPf14vX769O5UeDfA/X69+/PhAkTeOGFF3jrrbfw9vZ2c2LJr1wp/P8BHrLWnjLGeF607whQMftiiYgUDvFJKQz9YTOTVuwDoGn10ozsGkSV0kWx1hIXF0fx4sV57bXX6Ny5M61atXJzYsnvXB3cl5rO9jLAuavMIiJSqKzdd5LwiEj2RMfj4+nBgLuvo+ftNfH0MJw8eZJevXoRExPDTz/9RMWKFalYUf0ruXquFP7VwMPA3Mvs6wz8mS2JREQKuORUBx/9uoOxi3aQ6rBcX6EEo7sFU7eiPwDLli2jR48eHDp0iHfeeUfP5Uu2cqXwDwUWGGPmAJMAC9xhjOkDdAXuzIF8IiIFys7jcYRHRLLuQCzGQO87atK/1XX4eXuSmprKu+++y+uvv061atVYtmwZTZs2dXdkKWBcmbnvZ2NMV+ADoG3a5lHAIaCrtXZZDuQTESkQrLV8/cde3p2/mYRkB4ElizCiSxAh15b5u01cXByfffYZXbt2Zdy4cfj7+7sxsRRUrs7cN90YMwPnEr3lgWhgvbXWkRPhREQKgqOnExg4NYol208A0KlRIK/fXx9/P+fI/F9//ZVbb72VgIAAVq5cSbly5XR5X3KMSxP4AFinDdbaX621UeeLvjGmbvbHExHJ335Yd5i7R//Oku0nKFXUm08fbMSorsH4+3mTmJhIWFgYLVu25MMPPwSgfPnyKvqSo656vUZjzA3AKzgH+Gn9RxERIPZcMq/N2sDMyEMANK9Tjvc7N6S8v3Pes23bthEaGsratWvp168fzz77rDvjSiGSaaE2xtwL9ASqAjuAd6y164wxNYD3gY44J/P5MCeDiojkF8t3nGDA1CgOxyZQxNuTl9vW5cGbq/7dk581axYPPvggvr6+zJo1i/vvv9/NiaUwybDwG2MeAr7GOVPfLqAl0NoY8wjwDc4pe8cDQ621h3I4q4hInpaQnMrwhVv5fOluAIKqlGR01yBqliv+r3a1atXi9ttvZ+LEiVSuXNkdUaUQy6zH/xywBGhnrT1tjPECPgGmAQfTtmuVCBEp9DYeiiU8IpJtR+Pw9DA826I2z9x5LV6ezqFUf/31FzNmzODtt9+mfv36zJ8/382JpbDKbHBfPWCktfY0gLU2BXgT5y8ML6noi0hhl+qwfLJ4Bx3GLmPb0Thqli3G9Kea8dxdtfHy9MDhcDBq1ChCQkL46quvOH78uLsjSyGXWY+/CM55+C90OO2/27M/johI/rEvOp7+UyJZvfckAI+EVGNwm7oU8XEuZ3Ls2DEeffRR5s+fT4cOHfj8888pXbq0OyOLZGkUvk1ne3rz9ouIFGjWWqauPsAbczZyNimV8iV8ef+BhjSvU/7vNqmpqdx5553s3LmTsWPH8tRTT+kxPckTslL4pxljEi+zfeZF2621tk425RIRyZNOxCUyePp6ftp0FIB7G1RgaIcGlCrmA0BycjKenp54enoycuRIKlWqRMOGDd0ZWeRfMiv8U7h8j/+vHMgiIpKn/bzpKC9OX8eJuCRK+HrxZof6dAgO/Lsnv2fPHrp37063bt0ICwujdevWbk4scqkMC7+1NjS3goiI5FVnE1N4+4dNTF65H4CQmmUY0TWIwJJF/m4zdepUevXqhbWWSpUquSuqSKY0056ISAb+2htDeEQU+2Li8fH0YFDrOjx+aw08PJy9/Pj4eMLCwpg4cSI333wzkydPpkaNGm5OLZK+zCbwqW+t3ejKCY0x3kA1a+2Oq0omIuJGSSkOPvxlG58u3onDQt2K/nzQLZg6FUr8q92aNWv44osveOGFF3jrrbfw9vZ2U2KRrMmsx7/KGDMH+NRauzijhsaYCkAPIAwYC7yXLQlFRHLZ9qNnCIuIZOOh0xgDT/7nWsJb1cbXy/mYnrWWVatW0bRpU2677Ta2bdtGzZo13ZxaJGsyK/z1gaHAz8aYo8AyIAo4DiQCpYCaQFOgMXAIeB34vxzKKyKSYxwOy1d/7GHY/C0kpjioXKoIo7oG07TGP8/ex8TE0LNnT2bOnMmqVato3Lixir7kK5kN7tsN9DDGvAA8DtwDtAcuvJZ1GPgdGAbMsdbq+X4RyXcOx55j4NQolu2IBqBrk8q8cl89Svj988/d0qVL6dGjB4cPH2b48OHceOON7oorcsWyNLjPWrsfeAN4wxjjCZTDuUBPtLX2TA7mExHJcbMiD/LKzA2cTkihdDEf3u3UgHvqV/hXm/fee4+XXnqJ6tWrs3z5cm666SY3pRW5Oi6P6k/r0V88ja+ISL4TG5/MkFkbmBPlXFy05fXlGda5IeVK+F7S1tfXl9DQUD799FP8/f1zO6pIttHjfCJSKC3dfoKBU6M4cjqBoj6evHJfPUJvqvKvaXXnzp2Lw+Hg/vvv57nnngPQtLuS76nwi0ihkpRqeX32Rr5cvgeAG6uWZHTXYKqXLfZ3m8TERF544QU+/PBDmjdvTrt27VTwpcBQ4ReRQmP9gVheW36Ow2f34OVhCLurNk/+51q8PP9ZoXzbtm2Ehoaydu1ann32Wd577z0VfSlQVPhFpMBLSXUw7redfPDzdlIclmvLFeODbjfSoHLAv9rt2bOHRo0a4efnx+zZs2nXrp2bEovkHBV+ESnQ9pw4S/8pkazZdwqAVtW8+Kjn7fh5e/7dxuFw4OHhQfXq1Xnttdfo0aMHgYGB7ooskqM8Mm/yb8aYOsaYnsaYF4wx16Rtq2KMKZr98UREroy1lskr93HvmCWs2XeKCv5+/O+JpjxY1/dfRX/16tUEBwezcaNzdvLnn39eRV8KtCwXfmOMtzHmf8AmYALwDnD+b8fHwJDsjyci4rrjZxLp+dVqBk9fT3xSKu2CKrEw7A5ur13u7zYOh4ORI0fSrFkzTp06RVxcnBsTi+QeV3r8bwH3A72AasCFo13m4ZzVL1PGmNbGmK3GmB3GmBfTadPVGLPJGLPRGPOtC9EKEjsAACAASURBVBlFpJBbuPEI93zwO79sOYa/nxcfhgbzUfcbCSj6zwx8x44do23btgwcOJC2bdsSGRnJzTff7MbUIrnHlXv8DwKvWGu/SJu970K7gEzXoUw7bizQCjiAcxGg2dbaTRe0qQ0MBm611p40xpR3IaOIFFJnEpJ5a+4mpqw+AMCttcowoksQFQOKXNL2448/ZtGiRYwdO5annnpKo/alUHGl8JcDNmSw3y8L52gK7LDW7gIwxnyHc+7/TRe06QWMtdaeBLDWHnMho4gUQit3x9B/SiQHTp7D18uDF9tcz39DquPh8U9BT05OZv/+/QC8/PLLhIaGUq9ePXdFFnEbVwr/XuAm4NfL7GsCbM/COQKB/Re8PwBcfH3tOgBjzDLAE3jdWrvg4hMZY3oDvQHKlSvH4sWLs/DxcqXi4uL0HecCfc+uSXFYZmxPZt7uZCxQzd+D3g19CUzey++/7/273eHDh3n77beJjo7m448/5o8//gCcl/wl++n/47zNlcL/DfCyMWYHMCdtmzXGhAD9cQ72y8zlrqfZy2SqDTQHKgNLjDE3WGtP/esgayfgHGRInTp1bPPmzbP4Y8iVWLx4MfqOc56+56zbeuQMYRGRbD4cj4eBZ5rX4tmWtfHx+vfQpSlTpvDkk08CMHHiRMqWLavvOIfp/+O8zZXBfe8CvwBTgRNp2xYBS4HfgA+ycI4DQJUL3lcGDl2mzSxrbXLassBbcf4iICKCw2H5bMku2n20lM2HT1O1dFGmPhnCwHvq/KvoJyQk0KtXL7p160a9evWIjIyka9eubkwukjdkucdvrU0BOhpjWuEcwV8eiAYWWGsXZvE0q4DaxpgawEEgFOhxUZuZQHfgS2NMWZyX/ndlNaeIFFwHT51j4JQo/tgVDUD3plUY0rYexXwv/afM29ubPXv2MHjwYN544w28vb0vaSNSGGW58KeNro+21v4E/HTRPg+gbGYD8ay1KcaYvsBCnPfvv7DWbjTGvAmsttbOTtt3tzFmE5AKPG+tjXbppxKRAsVay8zIg7w6cyNnElMoW9yHYZ0acle9ay5p99lnn9G2bVsqVarE/Pnz8fLSBKUiF3LlUv9hoHE6+25M258pa+08a+111tprrbVD07a9mlb0sU79rbX1rLUNrLXfuZBRRAqYk2eT6PvtWsIjojiTmEKretewIOyOS4p+TEwMnTt3pnfv3nz66acAKvoil+HK34qMHnT1AhxXmUVE5F9+23ac56dGcexMIsV8PHmtXX26NKl8yXP3S5cupUePHhw5coQRI0YQHh7upsQieV+Ghd8YUxzwv2BTWWNMpYuaFcF5n/5oNmcTkULqXFIq787fzNd/OB/Ja1KtFKO6BlO1zKVLgkyfPp0uXbpQo0YNli9fTpMmTXI7rki+klmPfwDwatpryz+P8V3MAEOzK5SIFF5R+08RHhHJrhNn8fY0hLe6jj53XIunx+UvOt555508++yzvPHGG/j7+1+2jYj8I7PCPxc4grOwfwK8D+y+qE0isMlauzL744lIYZGS6mDsop2M+XU7qQ5L7fLFGd0tmBsCAy5pO2fOHMaOHcvs2bMpVaoUo0ePdkNikfwpw8Jvrf0L+AvAGGOB7621JzI6RkTEVbuOxxE+JYqo/c55unreVoOB99T51/K5AImJiQwaNIgxY8YQHBzMiRMnqFTp4ruPIpIRV57jH5+TQUSk8LHW8s2Kfbzzw2bOJadSMcCPkV2CaFar7CVtt27dSmhoKJGRkTz33HO89957+Pr6uiG1SP7m0rMuxpjrgMeAOly6KI+11rbNrmAiUrAdO53AoO/XsXjrcQA63hjI6/fXJ6DIpRPtWGt59NFH2b9/P3PmzOG+++7L7bgiBYYrE/g0BpbgHL1fFedUuqVxzuB3CNiXEwFFpOCZv/4wL81Yz8n4ZAKKeDO04w3c1/DSS/anT5/Gw8OD4sWL8+WXX1K8eHECAwPdkFik4HBlAp9hwA845803wEPW2grAfWnneSH744lIQXI6IZn+UyJ5atIaTsYnc3vtsiwMu+OyRX/VqlU0atSIfv36AVCnTh0VfZFs4ErhDwK+5J+JejzBORMfzpX53s/WZCJSoPy5K5o2Hyxh+pqD+Hp58Gb7+nz9eFMqBPz7rqHD4WDEiBE0a9aM5ORkevbs6abEIgWTK/f4fYEz1lqHMSYGuHC+zE1Aw2xNJiIFQmJKKiN/3MbEJbuwFhpWDmBU12BqlS9+Sdtjx47xyCOPsHDhQjp16sRnn31GqVKl3JBapOBypfDvAs5fj9sIPIrzOX+Ah4AMF+gRkcJn8+HThEdEsuXIGTw9DM+0qEW/FrXw9rz8xcb4+HjWrVvHp59+Sp8+fS6ZmldErp4rhX8+0Ar4DngXmJPW808BygADsz+eiORHqQ7LZ0t2MfLHbSSlOqhepiijugXTqOqlvffk5GQmTZrEf//7X6pXr87OnTspUqSIG1KLFA6uPMf/0gWvFxhjbgceAIoCC86vricihdv+mHgGTI1i5e4YAB68uSovt61LUZ9L/7nZvXs33bt3Z8WKFVSuXJm77rpLRV8kh13xmpXW2j+BP7Mxi4jkY9Zavl9zkNdnbyQuMYWyxX0Z/kBD7ry+/GXbR0RE0Lt3b4wxREREcNddd+VyYpHCyZVR/ekyxtQzxkzOjnOJSP4TczaJp75Zw8CpUcQlpnBP/Wv4MfyOdIv+Sy+9RGhoKPXq1SMyMpKuXbvmcmKRwivTHr9xjq5pgHPSnp3W2s0X7GuAc/W+jsC5nAopInnXoi3HeH7aOk7EJVLc14vX769P50aBGQ7MO9+7f+ONN/D2vnSmPhHJORkWfmNMBWA6cPMF274BngA+AJ4EknGu3KdleUUKkfikFIb+sJlJK5yTdjatUZqRXYKoUrroJW2ttXzyySecPHmSIUOG0KJFC1q0aJHbkUWEzHv8w4BgnEV9DVADGAT8BoQAEcDz1toDORlSRPKWtftOEh4RyZ7oeHw8PRh4z3U8cVtNPD0u7eXHxMTwxBNPMHPmTNq2bUtqaiqenp6XOauI5IbMCn8r4E1r7bDzG4wxG4CFwDhr7dM5GU5E8pbkVAcf/bKdsYt3kuqwXF+hBKO7BVO3ov9l2y9dupQePXpw5MgRRo4cSVhYGB4e2TK0SESuUGaFvzyw7KJt599rMJ9IIbLjWBzhEZGsPxiLMdDnjpr0v/s6fL0u33s/fvw4d999N5UqVWL58uU0adIklxOLyOVkVvg9gcSLtp1/fzb744hIXmOt5es/9vLOvM0kpjgILFmEkV2DuKVmmcu2j42NJSAggHLlyjF9+nSaNWuGv//lrwiISO7LynP8dxtjal3w3gOwQGtjzPUXNrTWfpud4UTEvY6eTmDg1CiWbD8BQOdGlXnt/nr4+11+JP7s2bN5/PHHGT9+PJ07d6Z169a5GVdEsiArhf/NdLa/fdF7C6jwixQQc9cd4uUZG4g9l0ypot6807EBbRpUvGzbhIQEBg0axEcffcSNN95IgwYNcjmtiGRVZoW/bq6kEJE8I/ZcMq/N2sDMyEMANK9Tjvc7N6S8v99l22/ZsoXQ0FCioqIICwtj2LBh+Pr65mZkEXFBhoXfWrs1t4KIiPst33GCAVOjOBybQBFvT15uW5cHb66a4WQ8q1at4uDBg8ydO5e2bdvmYloRuRJXPFe/iBQcCcmpDF+4lc+X7gYgqEpJRncNoma54pdtf/r0aVavXk2LFi14+OGHue+++yhV6tKV90Qk71HhFynkNhyMJTwiku3H4vD0MDzbojbP3HktXp6Xf95+1apVhIaGcvz4cfbu3UupUqVU9EXyEc2kIVJIpTosnyzeQcdPlrH9WBw1yxZj+lPNeO6u2pct+g6Hg+HDh9OsWTNSUlKYP3++Cr5IPqQev0ghtC86nv5TIlm99yQAj4RUY3CbuhTxufxkPCkpKbRr144FCxbQqVMnPvvsMxV9kXxKhV+kELHWMmX1ft6cs4mzSamUL+HL8C5B/Oe6chke5+XlxY033kj79u3p06dPhoP9RCRvu6LCnzahTxlgvbU2PnsjiUhOOBGXyODp6/lp01EA2jaoyNsdbqBUMZ/Ltk9OTuaVV16hffv2hISE8M477+RmXBHJIS4VfmPME8AbwPlZPG4C1hhjpgE/W2vHZXM+EckGP286yovT13EiLokSfl681f4G2gdXSrfnvmvXLrp3787KlSvx9fUlJCQklxOLSE7JcuE3xjwKTAAmAT8CX1+wewXQDVDhF8lD4hJTeHvuJr5btR+AkJplGNE1iMCSRdI9JiIigt69e2OMYcqUKXTp0iW34opILnClx/888KG1tr8xxpN/F/7NQP9sTSYiV+WvvTGER0SxLyYeHy8PBt1Th8dvrYGHR/r35+fOnUtoaCghISF8++23VK9ePfcCi0iucKXwXwv8kM6+M4CG+IrkAUkpDj78ZRufLt6Jw0Ldiv580C2YOhVKpHtMQkICfn5+tGnThgkTJvDoo4/i7X35hXhEJH9z5Tn+GKBKOvuuAw5ffRwRuRrbj56h4yfLGLtoJxZ4qvm1zHymWbpF31rL2LFjue666zh8+DCenp706tVLRV+kAHOlx/8DMMQY8wtwKG2bNcaUBMKAWdkdTkSyxuGwfLl8D8MWbCEpxUHlUkUY1TWYpjVKp3tMTEwMTzzxBDNnzuTee+/Fy0tP94oUBq78TX8Z+BPYBCzFuQzvCJwr+MXhHO0vIrnscOw5Bk6NYtmOaAC6NqnMK/fVo4Rf+r32JUuW0KNHD44ePcqoUaN47rnn8PDQRJ4ihUGWC7+19pgxphHOQX73AAeB0sBXwPvW2pM5E1FE0jMr8iCvzNzA6YQUShfz4d1ODbinfoVMjxs7dix+fn788ccfNG7cOBeSikhe4dK1PWvtKZw9/5dzJo6IZMWp+CRembWROVHOu24try/PsM4NKVfCN91jDhw4QHJyMjVq1GD8+PF4eHhQokT6A/5EpGDK8rU9Y8w7xpjrczKMiGRuyfbjtP5gCXOiDlHUx5N3OzXgs/82ybDoz5o1i6CgIB5//HEAAgICVPRFCilXbur1AzYaY1YbY/oZYzKe3FtEslVCciqvz97Iw5+v5MjpBBpVLcm8Z2+ne9Oq6c7Al5CQQL9+/ejQoQPVqlVj/PjxuZxaRPIaVy71lwc6AQ8Bo4CRxpgfcd7jn22tTcyBfCICrD8QS1jEWnYeP4uXhyHsrto8+Z9rL7t87nn79++nXbt2REVFERYWxrBhw/D1Tf+qgIgUDq4M7juHc7reScaYa4AH0/5EAKeNMVOttb1yJqZI4ZSS6uDTxTv58JftpDgstcoXZ3TXYBpUDsj02DJlylCqVCnmzp1L27ZtcyGtiOQHV/T8jrX2qLV2lLW2MdAS58x9j2drMpFCbs+Js3Qd/wcjf9pGisPy2K3VmdvvtgyL/unTp3n++eeJi4ujaNGi/Prrryr6IvIvV7osry/QAedl/7sBQ/rT+YqIC6y1fLdqP2/N3UR8UioV/P0Y0SWI22qXzfC4lStX0r17d/bu3csdd9xBu3bt0r33LyKFl6vL8jYHHgY6A/7AamAgMNlaeyLb04kUMqcSHfT8ajW/bDkGQLugSrzd/gYCiqY/GY/D4WDkyJG89NJLVKpUid9//51mzZrlVmQRyWdcWZZ3HxAI7AfGAl9ba7fmVDCRwmbhxiO8svQcZ5LP4e/nxVsdbqB9cGCmxw0ePJj333+fzp07M3HiREqV0npZIpI+V3r8P+Es9r/lVBiRwuhMQjJvztnE1L8OAHBrrTKM6BJExYAiGR6XmpqKp6cnTz31FLVq1aJnz566tC8imXJlVP8TORlEpDBauTuG/lMiOXDyHL5eHjxQy4u3HrkZD4/0C3hSUhKvvPIKmzdvZtasWVSvXp1evfRAjYhkTYaF3xjTFNhgrY1Pe50ha+3KbEsmUoAlpqQy+qftjP99J9bCDYH+jO4azMHNf2VY9Hft2kX37t1ZuXIlffr0ITk5GR8fn1xMLiL5XWY9/j+BW4CVaa9tOu1M2j7P7IsmUjBtPXKGsIhINh8+jYeBZ+6sxbMta+Pj5cHBzekf991339GnTx88PDyYNm0anTt3zr3QIlJgZFb42wDn/ym6l/QLv4hkwuGwfLFsN+8v2EpSqoOqpYsyulsQjauVzvTYM2fO0L9/f2644Qa+/fZbqlWrlguJRaQgyrDwW2sXXvB6Qc7HESmYDp46x8ApUfyxKxqA7k2rMKRtPYr5Zvy795YtW6hVqxYlSpTgt99+o0aNGnh5XdH0GyIigGur820yxjRIZ189Y8ym7IslUjBYa5mx9gCtR//OH7uiKVvch88eacK7nRpmWPSttXz88ccEBwczfPhwAGrXrq2iLyJXzZUpe68H0nu+qChQJysnMca0NsZsNcbsMMa8mEG7B4wx1hjTxIWMInnGybNJ9P12LeERUZxJTKFVvWtYGHYHd9W7JsPjoqOj6dixI/369aNly5b07NkzlxKLSGHgavchvXv8DYHYzA42xnjinPynFXAAWGWMmW2t3XRRuxLAs8AKF/OJ5AmLtx5j0LR1HDuTSDEfT15rV58uTSpn+pz9xo0befjhhzl69CijRo0iLCxMz+aLSLbK7HG+fkC/tLcWmGaMuXj53SJAJWBaFj6vKbDDWrsr7fzfAe2Bi28TvAW8j3M6YJF841xSKu/M28z//twLwE3VSzGySzBVyxTN0vFeXl4EBAQwc+ZMGjdunJNRRaSQyqzHfwj4K+11LWArEH1Rm0SchfvTLHze+Sl/zzsA3HxhA2PMjUAVa+1cY0y6hd8Y0xvoDVCuXDkWL16chY+XKxUXF6fvOBO7TqUyYV0iR+ItngY61famTY1Edq1fya4Mjjt+/DhLliyhU6dOBAYGMmbMGM6cOaPvO4fo/+Wcp+84b8tsVP/3wPfA+cuNL5/vrV+hy12z/Pv2gTHGAxgNPJrZiay1E4AJAHXq1LHNmze/iliSmcWLF6Pv+PJSUh2MXbSTMSu3k+qwXHdNcUZ3C6Z+pfSXzz1v1qxZPPnkkyQlJfHCCy+wfft2fc85TP8v5zx9x3lblgf3WWu7X2XRB2cPv8oF7yvjvKpwXgngBmCxMWYPzsmDZmuAn+RVu47H0XncH4z+eRupDkvP22owu+9tmRb9hIQE+vXrR4cOHahevTpr1qwhMDDzBXlERK5WZvf4B+FcmOdI2uuMWGvt8EzarAJqG2NqAAeBUKDHBSeIBf5edNwYsxgYaK1dncl5RXKVtZZvVuxj6A+bSEh2UCnAjxFdgmhWq2yWjr3rrrtYtmwZYWFhDBs2DF9f31xILSKS+T3+YcBi4Eja64xYIMPCb61NMcb0BRbinN73C2vtRmPMm8Bqa+3sLKUWcaNjpxMY9P06Fm89DkDHGwN5/f76BBTxzvA4a513tYwx9OvXj8GDB9O2bdsczysicqHMCn8Ra+35UfwZrxGaRdbaecC8i7a9mk7b5tnxmSLZZf76w7w0Yz0n45MJKOLN0I43cF/DSpkeFxsbS58+fWjZsiW9evWiW7duuZBWRORSmQ3uS7zca5HC5nRCMq/P3sj0NQcBuL12WYY/EESFAL9Mj12xYgXdu3dn37593HTTTTkdVUQkQ1mewMcYUxPwt9ZGpr33BV7EORhvobX2s5yJKOJef+6KZsCUKA6eOoeftwcv3VuXh2+plunEOg6Hg+HDhzNkyBACAwNZsmQJISEhuZRaROTyXJm57xOcz+tHpr1/CwgHtgEdjTGe1trx2ZxPxG0SU1IZ+eM2Ji7ZhbXQsHIAo7oGU6t88Swd/+eff/Liiy/SpUsXJkyYQMmSJXM4sYhI5lwp/MHAOADj7Oo8CrxkrR1ujHkbeBJQ4ZcCYfPh04RHRLLlyBk8PQzPtKhFvxa18PbM/AnYffv2UbVqVZo1a8by5cu55ZZbNO2uiOQZrizSUxI4kfY6GCgDTEl7/xNwbTbmEnGLVIdl/G87af/xMrYcOUONssWY9mQI/Vtdl2nRT0pKYtCgQdSqVYvVq51PoIaEhKjoi0ie4kqP/xhQE1iKc5Gd3dbavWn7igGp2ZxNJFftj4lnwNQoVu6OAeDBm6vyctu6FPXJ/K/Jrl27CA0NZdWqVTz55JPUr18/p+OKiFwRVwr/XGCoMeY6nHPkf3HBvvrA7uwMJpJbrLVM++sAb8zZRFxiCmWL+zL8gYbceX35LB3/3Xff0bt3bzw9PZk2bRqdO3fO4cQiIlfOlcL/Is4pdbsBPwNvX7CvK/BrNuYSyRXRcYm8NGM9CzceBaB1/Qq806kBpYv5ZPkcO3bsoEGDBnz77bdUq1Ytp6KKiGSLLBd+a+1p4OF09unhZMl3ft1ylEHT1nMiLpHivl68cX99OjUKzNI9+aioKE6ePEnz5s0ZPHgwL774Il5ervweLSLiHi7/S2WMKQE0BUrjXKJ3lbX2THYHE8kp8UkpDP1hM5NW7AOgaY3SjOwSRJXSRTM91lrL2LFjGTBgAPXq1WPNmjV4enrmdGQRkWzjUuE3xgzBecm/CP8ssRtvjHnXWjs0u8OJZLc1+07SPyKSPdHx+Hh6MPCe63jitpp4emTey4+Ojubxxx9n9uzZ3HvvvXz55ZcasS8i+Y4rM/c9A7wJTAK+wblwTwXgIeBNY0yMtfbTHEkpcpWSUx189Mt2Pl60A4eF6yuUYHS3YOpW9M/S8YcOHaJp06YcO3aM0aNH89xzz6noi0i+5EqPvy/wibW27wXbooCFxphYoB+gwi95zo5jcYRHRLL+YCzGQJ87atL/7uvw9cr6JfqKFSvSrVs3HnzwQRo1apSDaUVEcpYrE/jUBGals29W2n6RPMPhsHy1fA9txyxh/cFYAksWYXKvWxh8b90sFf39+/fTtm1bduzYgTGGkSNHquiLSL7nSo8/BqiDc5a+i9VJ2y+SJxyJTeD5aVEs2e6cbLJzo8q8dn89/P28s3T8zJkzefzxx0lOTmbr1q3UqlUrJ+OKiOQaVwr/TJwT+BwFpllrLYAxpiPOBXsm50A+EZfNXXeIl2dsIPZcMqWKevNOxwa0aVAxS8cmJCQwYMAAPvnkExo3bszkyZOpXbt2DicWEck9rk7g0wiIABKNMceAcoAvsCptv4jbxJ5L5rVZG5gZeQiA5nXK8X7nhpT398vyOd5//30++eQT+vfvz7vvvouPT9Yn8hERyQ9cmcAn1hjTDOgI3I7zOf4Y4DdglrVWc/WL2yzfcYIBU6M4HJtAEW9PXm5blwdvrpqlkffWWmJiYihTpgwDBw7k1ltvpWXLlrmQWkQk97n0HH9acZ+W9kfE7RKSUxm+cCufL3UuFRFcpSSjuwVTo2yxLB0fGxtLnz59WLt2LWvWrKFYsWIq+iJSoGVa+I0xoUAYUAs4hfNS/2vW2pQcziaSoQ0HYwmPiGT7sTg8PQzPtazN082vxSuT5XPPW7FiBd27d2ffvn289dZb+Pll/ZaAiEh+lWHhN8Z0Ab4F9gHLgBo47+V7onv64iapDsu433bywc/bSE611CxbjNHdggmqUjJLxzscDoYPH86QIUMIDAxkyZIlhISE5HBqEZG8IbMef3/gB6CTtTYZwBjzDvCcMeYla60jpwOKXGhfdDz9p0Syeu9JAP4bUo0X29SliE/WJ+NJTU1l1qxZdOzYkQkTJvx/e/cdH1WVPn788yQhBDAhQOhVQHoJCAi7CvJTsSEIQiDIuoqILuUrKrAo6qIiIh0UXSy4lgWUIk3AVYQVFVRKQlPWBIEAgrQAIXWS8/vj3ugwpsxMZlKf9+s1r2TuPXPmyZnJPHPPPfccwsPd+8KglFKlQX6JvznwQnbSt83HOtpvCPzsr8CUcmaM4aMdCTy/9gCX0zOpEVqeGQPb06NZdbfr+Oyzz+jQoQMRERFs3LiR0NBQnXZXKVXm5HcyNBw447LttP2ziu/DUeqPziSl8dB7O/n7ir1cTs/kzra1+XRsd7eTfnp6OuPHj6dXr15MmTIFgLCwME36SqkyyZ1R/cbD7Ur5zGcHTjFxxR7OXk4nNCSIF/q2oW9kHbeTdnx8PNHR0Xz//fc88sgjvPTSS36OWCmlijd3Ev/XuXzIfuuy3RhjyvskKlXmJaU5mLLuAEu/TwCgW+NqzIxqT93wCm7X8cUXX3D33XcTGBjI8uXLueeee/wVrlJKlRj5Jf6XCyUKpZzsPHKOxz6M5ei5ZIKDAphwa3OG/flqAgI865pv06YNt9xyC7Nnz6Zhw4Z+ilYppUqWPBO/MebJwgpEqXRHFvM2/Y/Xt8STZaBl7TDmDoqkea1Qt+uIiYlh3rx5vPnmm9SoUYMVK1b4MWKllCp5PJq5Tyl/+enUJcZ+GMP+ExcRgb/d2ISxN1/j1vK5YI36f+WVVxg/fjwREREcPnxYV9RTSqkcaOJXRSory/Cvbw4zbeOPpDuyqFelAnMGRdK5UVW36zhz5gzDhg1j7dq19O7dm3feeYeIiAg/Rq2UUiWXJn5VZE4kpjB+eSxfx50FIKpTPZ7p3YrQkHIe1TNgwAC2bdvGvHnzGDNmjF6mp5RSedDEr4rE6pjjPL1qH5dSHVSrFMxL/dvSq3Uttx/vcDjIzMykfPnyzJkzB4AOHTr4K1yllCo1NPGrQpWYnM4zq/ezNvYEADe1qMG0e9pRPdT9K0ETEhIYMmQI7dq1Y8GCBZrwlVLKA5r4VaHZ+tNpxi/bw8mLqVQMDuTZ3q0Y1Lm+R13zq1atYtiwYWRkZPDII4/4MVqllCqdPEr8IlITeBToDlQFBhhjDojISOA7Y8wOP8SoSriU9Exe3vgj//rmMAAdG4QzZ1AkDatVcr+OlBTGjRvHa6+9xrXXXsvSpUt11L5SSnnB7cQvIi2AL4FywPdANyB7AfPmwJ+Aob4OUJVse49dYOyHu4k/fZmgAOGxW5rxjSrqbgAAIABJREFUcPfGBAXmt0zElY4fP857773HE088wdSpUwkODvZTxEopVbp5csQ/E2s1vluBJCDdad/XgE6Crn7jyMzi9S3xzNv0E44sQ9MaVzF3UCRt6lZ2uw5jDJs2beKmm26iadOmxMXFUbNmTT9GrZRSpZ8nh109gKnGmET+uEDPSaC2z6JSJdrhM5cZuHAbsz77H44swwN/bsS6Mdd7lPQvXLhAdHQ0t9xyC+vWrQPQpK+UUj7g6eC+zFy2VwNSChiLKuGMMSz5LoEX1h0gJSOTWmEhzBzYnuuv8Wwyne3btxMdHU1CQgJTp07lzjvv9FPESilV9niS+HcAfwHW5bDvHmC7TyJSJdKvl1KZuGIvX/z4KwB92tfhhb5tqFzRs8l4FixYwNixY6lbty5bt26lW7du/ghXKaXKLE8S/4vARhFZC/wbq7u/u4g8DEQBPf0QnyoBPt1/kidX7uXc5XTCQoJ44e429I2s61VdDRs2pH///ixcuJDw8HAfR6qUUsrtxG+M+VxEooC5QHbf62zgBBBljPnaD/GpYuxSagbPrz3Asp3HALi+aQQzBrajduUKHtWzceNG4uPjGTVqFL1796Z3797+CFcppRQenuM3xqwUkY+B1kAN4Cyw1xiT5Y/gVPH13c/nePyjGI6dT6F8UABP3t6C+7o1IiDA/cl40tPTeeqpp5g1axYdOnRgxIgRlCvn2akBpZRSnvF45j5jjAH2+SEWVQKkOTKZ89lPLPwyHmOgTd0w5kRFck3NUI/qiYuLIzo6mh07djBy5EhmzpypSV8ppQqBJxP4ROVXxhjzUcHCUcXZwZOXGPthDD/8cpEAgVE9m/J/N11DcJBnk/EkJibSpUsXjDGsXLmSfv36+SlipZRSrjw54l+ay3bna/o18ZdCWVmGDT9n8PFnX5GemUXDahWZHdWeaxtW9ageh8NBUFAQ4eHhzJ8/n+7du9OgQQM/Ra2UUionnhyqtczhdj3wMhBv/65KmeOJKQx5azsfHkwnPTOL6C4NWP9/N3ic9GNiYmjbti0bN24EYOjQoZr0lVKqCHgyqv9gLru+EZFM4G/ANp9EpYqcMYZVMcd5dtV+LqU5CAuGOdGduKmlZ7PnGWN45ZVXGD9+PBEREVSsWNFPESullHKHr5bl3Qys9FFdqoidv5zOpFV7Wb/3JAC9WtWkd61LHif9M2fOMGzYMNauXctdd93FokWLiIjwbBY/pZRSvuXZqKzcdQKSfVSXKkJbDv7KrXO/ZP3ek1QKDmT6gHYs/Mu1hAW7f5letjVr1vDpp58yb948Vq9erUlfKaWKAU9G9U/IYXMw0AboB7zpq6BU4UtJz2Tq+h94f/sRADo3qsLsqEjqV/Wsa97hcLB//37at2/PAw88QPfu3WnatKk/QlZKKeUFT7r6p+WwLRM4DswBnvNJRKrQxSQk8viHMRw6c5lygcLjtzRnRPfGBHowGQ/A0aNHuffee4mNjSUuLo4aNWpo0ldKqWLGk8Sf0zysGTprX8mVkZnFgs1xvPJFHJlZhmY1r2LOoEha13F/+dxsH3/8MQ8++CAOh4PXX3+dGjVq+CFipZRSBeVW4heRYGAysNwYs9OvEalCceh0Eo99FEtsQiIAw6+/mnG3NiekXKBH9WRlZTF69Ghef/11OnXqxNKlS2nSpIk/QlZKKeUDbiV+Y0y6iDwKbPBzPMrPjDF88O1RXvzkAKkZWdSpHMLMge35U1PvBt4FBATgcDh44oknmDp1KsHBwT6OWCmllC950tUfC7QCvvRTLMrPfr2YyoQVe9hy8DQA/TrUZXKf1lSu4Nkc+cYY3nrrLTp16kSHDh1YuHAhIp6P+ldKKVX4PLmcbwLwdxG5uSBPKCK3ichBEYkTkYk57H9cRA6IyB4R2SQiDQvyfMqyYe8v9Jr7JVsOnia8YjkWDOnInEGRHif9xMREBg0axIgRI1i4cCGAJn2llCpBPDniXwSEA5+KSDJwkivn6TfGmOZ5VSAigcAC4BbgGPC9iKwxxhxwKrYb6GSMSRaRvwHTgUEexKmcXEzNYPKa/azcdRyAG66JYObA9tQMC/G4rm3bthEdHc3x48eZNm0a48eP93W4Siml/MyTxL+TKxO9N7oAccaYQwAishToC/yW+I0xm53KbweGFvA5y6xt8WcZtyyW44kphJQL4Kk7WvKXrg29OkKPiYlh3Lhx1K9fn61bt9K1a1c/RKyUUsrfPJmrf7APnq8ukOB0/xhwXR7lH0QHFHosNSOTWf85yFtf/Ywx0L5eZWYPiqRJ9as8rssYg4jQpk0bJk2axGOPPUZ4eLgfolZKKVUY8kz8InII6GeMifXR8+V0qJljL4KIDMWaCrhHLvtHACMAqlevzpYtW3wUYsmWcCmLhbGpHEsyBAj0aVKOu5pkkLB/xxXfuNzx7bff8vbbbzNjxgwCAwPp2bMnMTExfolbWZKSkvS97Gfaxv6nbVy85XfE3wgo78PnOwbUd7pfDzjhWsgeQDgJ6GGMScupImPMG8AbAM2bNzc33nijD8MseTKzDG9tPcSs7f8jPdNwdUQlZke1p0ODKh7XlZ6ezpNPPsns2bNp27Ytbdq0ISEhgbLexoVhy5Yt2s5+pm3sf9rGxZuvVudz1/fANSJyNdZUv4OBIc4FRKQDsBC4zRjzayHHVyIlnEvmiY9i+e7wOQCGdm3AU3e0pGKw5y9vXFwc0dHR7Nixg5EjRzJz5kwqVKhAQoKn/QVKKaWKI3cyQ0EH9P1ekTEOERkNfAoEAouMMftF5HlghzFmDTADuApYZg9CO2qM6eOrGEoTYwzLdx7jubUHSEpzUD20PNMHtKNnc++ny33qqaeIj49n5cqV9OvXz4fRKqWUKg7cSfzPicgZN8oZY8xf3Si0Hljvsu1Zp98LNE9AWXE2KY2nPt7Lp/tPAXBb61pM7d+WqpU8nzkvKSmJpKQkatWqxauvvkpqaioNGjTwdchKKaWKAXcSfySQ43l2Fz7rGVB5++LHU0xYvpczSWlcVT6I5/q0pn/Hul5dprd7924GDx5M7dq12bx5sy6uo5RSpZw7if9uY8x3fo9E5etymoMX1//A4m+PAtDl6qrMGtie+lUrelyXMYb58+czYcIEqlevznPPPacz8CmlVBlQ2IP7lJd2HT3P4x/GcPhsMsGBAYy7tRkPXt+YwADPk/W5c+e4//77Wbt2LXfddReLFi0iIsK7RXqUUkqVLJr4i7mMzCxe2fQTr26OI8tAi1qhzBkUScvaYV7XGRQURHx8PPPnz2f06NF6pK+UUmWIJv5iLO7XJB77MIa9xy8gAg93b8zjvZpRPijQ47ocDgcLFizg4YcfJiwsjJiYGMqV82yBHqWUUiVfnonfGOPJ6n3KR7KyDO9vP8LU9T+Q5siibngFZke157rG1byq78iRIwwZMoRvvvmGatWqMXToUE36SilVRukRfzFz8kIq45fHsvUn6wrKezrWY3KfVoSGeJeoV6xYwfDhw8nMzGTx4sVER0f7MlyllFIljCb+YmRt7AmeXrWPCykZVKlYjpf6t+W2NrW9rm/GjBlMmDCBzp07s2TJEpo0aeLDaJVSSpVEmviLgQvJGTy7Zh+rY6xlC3o2r87LA9pRIzSkQPX26dOH8+fPM3nyZIKDPZ/YRymlVOmjib+IfR13hnHLYvnlQioVygXydO+WDOnSwKuR9sYY3nzzTbZv387bb79N8+bNmTp1qh+iVkopVVJp4i8iqRmZTN94kEVf/wxAZP1w5gyK5OqISl7Vl5iYyIgRI1i2bBk333wzKSkpVKzo+cQ+SimlSjdN/EVg3/ELPPZhDD/9mkRggPDoTdcw8sYmBAV6dxHFtm3biI6O5vjx40ybNo3x48cTEKAXZCillPojTfyFKDPL8M//xjP38/+RkWloXL0Sc6IiaV8/3Os6U1JS6NevHxUrVuSrr77iuuuu82HESimlShtN/IXk6NlkHv8ohh1HzgPw124NmXh7SyoEez4ZD8Dp06epVq0aFSpUYM2aNTRv3pzKlSv7MmSllFKlkPYH+5kxhg+/P8rt875kx5Hz1Agtz7vDuvBc3zZeJ/0NGzbQunVrZs2aBUCXLl006SullHKLHvH70ZmkNCau2MvnP5wC4M62tXmxXxvCK3p3aV16ejpPPvkks2fPpl27dvTu3duX4SqllCoDNPH7yWcHTjFxxR7OXk4nNCSIF/q2oW9kHa8XxImLi2Pw4MHs3LmTUaNGMXPmTEJCCnadv1JKqbJHE7+PJaU5mLLuAEu/TwCgW+NqzIxqT93wCgWq95dffuHo0aN8/PHH3H333b4IVSmlVBmkid+Hdhw+x+MfxXL0XDLBQQFMuLU5w/58NQEB3h3lJyUlsX79eqKiorjhhhv4+eefqVTJu+v8lVJKKdDE7xPpjizmbfofr2+JJ8tAq9phzB0cSbOaoV7XuWvXLgYPHsyhQ4fo1KkTjRs31qSvlFKqwHRUfwH9dOoS/V77mgWb4zHA325swqpRf/Y66RtjmDt3Ll27diU5OZlNmzbRuHFj3watlFKqzNIjfi9lZRne+eYwL2/8kXRHFvWrVmB2VCSdG1X1uk5jDFFRUSxfvpw+ffqwaNEiqlWr5sOolVJKlXWa+L1wIjGFccti+Sb+LACDOtXnmbtacVX5gjWniNCrVy969OjBqFGjvL4CQCmllMqNJn4PrY45ztOr9nEp1UG1SsG81L8tvVrX8ro+h8PB5MmTad26NdHR0Tz00EM+jFYppZS6kp7jd1NicjqjF+/i0aUxXEp1cFOLGmwc271ASf/IkSP06NGDF198ke3bt/swWqWUUipnesTvhp1HzjHy37s4dTGNisGBPNu7FYM61y9QV/yKFSsYPnw4mZmZLF68mOjoaB9GrJRSSuVME38+TiSmMPzdHZxPzqBjg3DmDIqkYbWCXVa3c+dOBgwYQOfOnVmyZAlNmjTxUbRKKaVU3jTx5yEjM4sxS3ZzPjmD7s2qs+ivnQgK9P7syKVLlwgNDeXaa69l2bJl9OnTh+Bg7+btV0oppbyh5/jzMOPTg+w8cp5aYSHMHRTpddI3xvDGG2/QsGFDYmNjARgwYIAmfaWUUoVOE38uPj9wije+PERggPDqkA5UreRdkk5MTCQqKoqHH36YTp06UbNmTR9HqpRSSrlPE38OEs4l88Qy68h8wq3N6eTlpDzbtm0jMjKSVatW8fLLL7Nx40Zq1fL+KgCllFKqoPQcv4t0Rxajl+zmQkoGN7WowUM3eD9d7po1awgICOCrr77iuuuu82GUSimllHf0iN/FSxt+IDYhkbrhFZgV1d7jlfVOnDjBzp07AXj++efZvXu3Jn2llFLFhiZ+Jxv3/cI7Xx+mXKB1Xj+8omfn9T/55BPat2/PkCFDyMzMpFy5clSuXNlP0SqllFKe08RvO3L2MuOX7QFg4u0t6dCgituPTUtL47HHHqN3797UqVOH1atXExgY6K9QlVJKKa/pOX4gNSOTUYt3cSnNwa2tazLsz43cfuzZs2fp1asXu3btYsyYMUyfPp2QkBD/BauUUkoVgB7xAy9+8gP7jl+kftUKTB/Q3qOpeKtUqULLli1ZtWoV8+fP16SvlFKqWCvziX9t7Ane336E4MAAXhtyLZUrlMv3MZcuXWLkyJEkJCQQEBDABx98QN++fQshWqWUUqpgynTiP3Q6iYkrrPP6z/RuSdt6+Q/E27lzJx07dmThwoVs3rzZ3yEqpZRSPlVmE39qRiYj/72Ly+mZ9G5Xm6FdG+ZZ3hjDnDlz6NatGykpKWzevJn77ruvkKJVSimlfKPMJv7Ja/bz48lLXB1RiZf6t833vP7s2bN5/PHHueOOO4iNjaV79+6FFKlSSinlO2VyVP/KXcdY+n0C5YMCWDCkI6EhuZ/XT09PJzg4mIceeogqVarwwAMPeDT4TymllCpOytwR/0+nLjHp430ATO7TmlZ1wnIs53A4mDRpEl27diU1NZWwsDCGDRumSV8ppVSJVqYSf3K6g5H/3kVKRib9OtRlcOf6OZY7cuQIPXr0YOrUqXTo0IGsrKxCjlQppZTyjzLT1W+M4elV+/jp1ySaVK/ElLvb5Hj0vmLFCoYPH05mZiaLFy8mOjq6CKJVSiml/KPMJP5lO46xctdxQsoF8Nq911Kp/B//dIfDwZQpU2jWrBlLliyhcWPvV+ZTSimliqMykfh/PHmRZ1Zb5/Wn3N2W5rVCr9i/f/9+6tWrR+XKlfnkk0+oXr065crlP5GPUkopVdKU+nP8SWnWef00RxYDr63HgGvr/bbPGMPChQvp1KkTEydOBKBOnTqa9JVSSpVapTrxG2N4auVeDp2+TPOaoTzft81v+86fP8/AgQN55JFH6N69O5MnTy66QJVSSqlCUqoT/+LvjrIm9gQVgwNZcG9HKgRbS+Xu3r2byMhIVq9ezfTp09mwYQM1a9Ys4miVUkop/yu15/j3Hb/Ac2sPAPBS/7Y0rXHVb/uqV69OrVq1WLZsGV26dCmqEJVSSqlCVyqP+C+mZjBq8S7SHVkMua4BfSPrcuLECSZNmkRWVhb16tVj+/btmvSVUkqVOaUu8RtjmLhiD0fOJtOqdhjP9m7FunXraNeuHXPnzmXfPmt0v87Ap5RSqiwqdYn/3W8Os37vSa4qH8ScAa2ZOP4J7rrrLurVq8fOnTtp165dUYeolFJKFZlSdY4/NiGRF9f/AMDL97Tj76OGsXr1asaMGcP06dMJCQkp4giVUkqpolVqEv+FZOu8fkam4b7r6nNnu9pUHjeOBx54gL59+xZ1eEoppVSxUOhd/SJym4gcFJE4EZmYw/7yIvKhvf9bEWnkTr3jlsdy9ORZHF/MJ/mb9wG4/vrrNekrpZRSTgo18YtIILAAuB1oBUSLSCuXYg8C540xTYE5wMv51XsxzbDui6859d5Yftn5OWGhofk9RCmllCqTCrurvwsQZ4w5BCAiS4G+wAGnMn2Byfbvy4FXRUSMMSa3Sk+fO0/m++OIqF6DFZs30717d/9Er5RSSpVwhd3VXxdIcLp/zN6WYxljjAO4AFTLq9LMS2dp0bk7P+7fo0lfKaWUykNhH/HndPG865G8O2UQkRHACPtu2o/ffrEvIiKigOGpPEQAZ4o6iDJA29n/tI39T9vY/5p7+8DCTvzHgPpO9+sBJ3Ipc0xEgoDKwDnXiowxbwBvAIjIDmNMJ79ErABt48Ki7ex/2sb+p23sfyKyw9vHFnZX//fANSJytYgEA4OBNS5l1gB/tX8fAHyR1/l9pZRSSrmvUI/4jTEOERkNfAoEAouMMftF5HlghzFmDfA28L6IxGEd6Q8uzBiVUkqp0qzQJ/AxxqwH1rtse9bp91RgoIfVvuGD0FTetI0Lh7az/2kb+5+2sf953caivehKKaVU2VHqFulRSimlVO5KVOL313S/6ndutPHjInJARPaIyCYRaVgUcZZk+bWxU7kBImJEREdHe8GddhaRKPv9vF9EFhd2jCWdG58XDURks4jstj8z7iiKOEsyEVkkIr+KyL5c9ouIzLdfgz0i0jHfSo0xJeKGNRgwHmgMBAOxQCuXMiOBf9q/DwY+LOq4S9LNzTbuCVS0f/+btrHv29guFwp8CWwHOhV13CXt5uZ7+RpgN1DFvl+jqOMuSTc32/gN4G/2762Aw0Udd0m7Ad2BjsC+XPbfAWzAmgOnK/BtfnWWpCP+36b7NcakA9nT/TrrC7xr/74cuElEcpoQSOUs3zY2xmw2xiTbd7djzcWg3OfO+xjgBWA6kFqYwZUi7rTzQ8ACY8x5AGPMr4UcY0nnThsbIMz+vTJ/nLdF5cMY8yU5zGXjpC/wnrFsB8JFpHZedZakxO+X6X7VFdxpY2cPYn3TVO7Lt41FpANQ3xizrjADK2XceS83A5qJyNcisl1Ebiu06EoHd9p4MjBURI5hXc01pnBCK1M8/dwu/Mv5CsBn0/2qXLndfiIyFOgE9PBrRKVPnm0sIgFYq1LeX1gBlVLuvJeDsLr7b8TqudoqIm2MMYl+jq20cKeNo4F/GWNmiUg3rDla2hhjsvwfXpnhcd4rSUf8nkz3S17T/apcudPGiMjNwCSgjzEmrZBiKy3ya+NQoA2wRUQOY52zW6MD/Dzm7ufFamNMhjHmZ+Ag1hcB5R532vhB4CMAY8w2IARrHn/lO259bjsrSYlfp/v1v3zb2O6GXoiV9PWcqOfybGNjzAVjTIQxppExphHWOIo+xhiv5+Uuo9z5vFiFNVgVEYnA6vo/VKhRlmzutPFR4CYAEWmJlfhPF2qUpd8a4D57dH9X4IIx5pe8HlBiuvqNTvfrd2628QzgKmCZPW7yqDGmT5EFXcK42caqgNxs50+BXiJyAMgExhtjzhZd1CWLm238BPCmiDyG1f18vx6MeUZElmCdjoqwx0r8AygHYIz5J9bYiTuAOCAZeCDfOvU1UEoppcqOktTVr5RSSqkC0sSvlFJKlSGa+JVSSqkyRBO/UkopVYZo4ldKKaXKEE38SimlVBmiiV8VOhG5315uNqfbzR7WNdx+XKEsFiQiU1ziPW8vAe3zOSNEJMh+jqedtvUXkbE5lL3ZLnu9r+PII76mLm2RKSK/iMj7IpLnXOF51NlRRCaLSLgf4m0gIskiEum07YPc3ote1L9URH50ut/Crsut94aIVBKRp0Vkn4ikiEiiiGwRkYGexuJUZ1O7PRu4bA8QazlinTu/DCoxE/ioUmkg1nSTzg4URSBe6Gb/rAY8DCwRkWBjzHu+egJ7gpRuXLkAR3/gemCuS/Hv7Jj2++r5PTAF+AQob8fwLNBCRLrZi2V5oiPWBCX/Anw9Z/4U4D/GmBiX7SeBfj5+Lo+ISFVgE9YStzOBr4CKWP8jH4nIfGPMo15U3RSrPT/HmkUPAGNMloi8AMwXkXeNMRcL+jeokkMTvypKMcaYuKIOwhv28pcAiMh/sOZ5Hwv4LPG7Pk8+5S5iTe9bFOKd4vyviJTHWpUtEigWUw2LSB1gCHBnDrvT3G1nP3oNaAF0NcbEOm3/RER+AKaJyDfGmA99+JzLgVewFoSa78N6VTGnXf2qWBKRCiIyT0T2i8hluwt5jYg0d+OxfxGRGPtxF0Rkj4gMdynTU0S+EJEk+7ZBRFp5E6sxJgOIwTq6yq6/soi8ZsedLiIHReSKIzYRCRORV0UkQUTSROSUiHwmIs3s/Vd09YvIB8C9QEOnLuk4e98VXf0i8oaInBCRQJfnDLHbZKbTthoistAuny4iP4jIg960hW2X/dO1e3mKiOwWkYsickZENolIF6f9w4E37bs/O/2N9ZzaY5LdlmkiclxEZthfNPLzANY03p97+sfYXfb/FpHDdhd8vIi8IiJh+T/arfqvBqKA11ySfrYZWNOxTnR6zDQRSc2hrt9ON4i1zHD2stlbndqzK/z2vl0JDHetR5VuesSvilKgWKsoZjPGmEz79wr27XmsrthqwChgm4i0yG2BIBHpAbyL1RX+BNYc4q2AKk5l+gIrsBa3GIL1BXgi1odjO2PMcS/+lquxu6btZLsBaAc8g9X93geYKyLVjDHP2o+ZB9yGtdJhHNaqZddjrSqZk3/YZdrze9f0Hz78be8BD2EtkPIfp+19gTDgfTvWcOBrrLm/nwUOY837/aZ96uJ1t/76KzWyf8a7bK8DzMI6vXMV1oJaW0WkozFmP7Aaq6v7SaxTGtkLjWS/1kuA24FpWL0brbHeHw2AQfnEdBvwjdP76wou70OALKelY+tiLd7zEdZr3BTrNWuLNYd6QfXEWlo1x3Ua7G75T4BHRaSqMcbdFUe3AY9hLfP8MLDH3r7PqcyXwEMiUscYk+eKbqoUMcboTW+FesPqWjQ53L7K4zGBQCWsRSjGOG0fbj+2nn1/IvBrHvUIVnL71GV7ONYR4cx8Yp9iP1+QfasJvGBvm2mXudu+P9Tlsf/CStRV7fs/AtPzeK4gu56nnbZ9ABzOoezNdtnrnf7OQ8D7LuXWAXuc7j8HpABNXMq9A5wCAvOIr6n9nMPsWCthfdE4ASzNpx0Dsb5sxAOzcng9G7mU72lvH+Ky/a/29rZ5PFeA3e7P5bDvg1zei5PzeV2y27ul0/alwI9O91vYZQbn0xb/sMs1zKPMWLtMO/v+NCA1h3KuMdzm/L7IoXxre39/d/9/9Vbyb9rVr4pSP6Cz0+2K7mURGSwi34nIBcABJGH1AuTV3f89UF1E3hORO0XE9ei5BdAQ+LfddRxkH+0lAd8C3d2MPcO+nQTGA7OxjgKx63BgfQg7+wBrANx1TrE+KCITReRaEfHZ/6OxPtU/APqJSCUAEakO3MqV4xBuA74Bjri0x6dADfJu62xvY7VFElZX+jF+Xx77NyLSS6xR6mex2icd6wjfnee4DSt5f+wSZ3Zvxg15PLYaVrvnthzsL1z5PuwMvOEUd4iIPGOfYki1/9bP7N3uxJ5djzjH7nQaRtx5uLvP46HsNqnjp/pVMaSJXxWlfcaYHU63g9k7RKQfVtfuPiAaK1l2xjoqD8mtQmPMJqxu30ZY662fEZH/iEgbu0gN++e7/J68s2+3YSUJd2QniKZAqDHmCWNMmr2vKnDG/HFE+0mn/QAjsc5pP4Q1CO5XEZklIhXcjCE/72Edhfe370dj/c8vdipTA/h//LEtltj73WmP57Da4kbgdfv3V5wLiEhnrJH/F7B6CLra5faRx+vpEmcIVo+Pc5zZ3dN5xZldf1ou+9Nd3oc7zJXd3rOAp7F6bG4HuvD7kt/uxJ7tYZfYs6/AyL5qo1Eej21o/3S9CqagUuyfvnrPqRJAz/Gr4mowVpflsOwNIhKC1SWfJ2PMR1iXQF2FldReBjaIdS1z9nrrE4DNOTw8t+Tg+hx5jVY/h7V2dpBL8q9l/zxr13EJ69TERBFphHXp1ktYR7aTKCBjTJyIbAeReuJQAAAENUlEQVSGYp3THwpscklqZ7ESz+O5VHMwl+3ODju1x3/tQW/DReSfxpjsgX4DsP6ue5zbRKzL2E658RxnsZJ+j1z253V+Ovs1r5JHmbwMAt40xryUvUFEIryoZwVXXuWQnXQ3Y3W39wH+6/oguyfoTqyrYLLP76cCQSISYH4fiwDuf3HNlv0l9IyHj1MlmCZ+VVxVxOoOdnYfHvRSGWOSgDUi0hTrqK0K1jwBCUArY8wMH8Xq6r9Yg6ruAZwvv7oX6wP72xxiPQzMEJG/AG1c9ztJw7Ojs/exrtXuiXWE/ReX/RuxjkQPG2N89eH/d6y//R9Ygwnh99fzt4lxRKQXVhfzD06Pzf7i5fo3bsQarFnJGPOH5JgXY0yyiCRgnVbwiIiIHUuGy64HPK3LGHOaHE43GGMOicgKYKSIvGf+OLJ/PFbP0hCnbUewxkm0wJ77wv4y0oXfB0VC7u2Z7Wr7pztf8FQpoYlfFVcbgVfty842YCWtUUCeE42IyItYRz2bsT4AGwCjgR3ZR0siMhpYafcgLMM6IqwF/Ak4ZIyZV8DY12GNqH5TRGphJbbeWIMaXzDGnLfj+Bbrcqp9wGWsAWytgYV51H0AGCYiI4DdQIoxZl8e5Zdijep+336Oj132z8TqadgqInOA/wGhWAnlT8YYjye2McYcF5F/AmNFJNJYE+ZsxHod3hGRd+36n+aPR+rZEziNFuvyxQwg1hjzuYgswzrHPxtrwiKwusfvAJ4wxrheReDsS6yk6OnfYsSap2G4fZncYaxL7zp6Wlc+HsF6z/7Xfs9/jZWsB2KNl3jNGLPEqfxarNdzkYg8j3VKZyLWqRRnPwJZdvyXscZV/GCMuWzvvw6r56FYzLegCklRjy7UW9m78fuo/qZ5lAkEpmIlhmSsD8X2WOc433Iq5zqqvw/WgK9fsI52ErDOo9dyqf/PWOecz2Mdhf+MdV67az6xT8EeO5dPucpYk7L8gvVhexB41KXMTKzkfQFrYNweYLTT/pxG9Ydi9SKct/fF2duvGNXv8jwf2/veyyXWqliXFh62Y/0VK1GOyedvzB7Vf38O+2rYf9MKp21j7edIwUrcPbFmqPvc5bHP2697pstrG4jVk7LHfs0SseZPeBkIyyfWu7ASYH2X7TleJeFSpibWF8RErNM472J9SbxixD5ejup3Kn8VVi/JfruNLmL1Hg3KpXxPrDkTkrG+XEa5xmCXG223e3aPS1enfVuBDwrrf19vxeMm9ouvlFKllj2CPh74pzFmWlHHUxzY40oOATcYY74u2mhUYdLEr5QqE0Tkr1jXvzc2xqTkV760E5HXseYOuKOoY1GFS8/xK6XKivewxnI0xDr3XWbZVwocxZp/QpUxesSvlFJKlSE6gY9SSilVhmjiV0oppcoQTfxKKaVUGaKJXymllCpDNPErpZRSZcj/BxMN6C/bA42BAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_roc_curve(fpr, tpr, label=None):\n",
    "    plt.plot(fpr, tpr, linewidth=2, label=label)\n",
    "    plt.plot([0, 1], [0, 1], 'k--') # dashed diagonal\n",
    "    plt.axis([0, 1, 0, 1])                                    # Not shown in the book\n",
    "    plt.xlabel('False Positive Rate (Fall-Out)', fontsize=16) # Not shown\n",
    "    plt.ylabel('True Positive Rate (Recall)', fontsize=16)    # Not shown\n",
    "    plt.grid(True)                                            # Not shown\n",
    "\n",
    "plt.figure(figsize=(8, 6))                                  # Not shown\n",
    "plt.title('AUROC')\n",
    "# plt.legend(loc='upper left')\n",
    "plot_roc_curve(fpr, tpr)\n",
    "# fpr_90 = fpr[np.argmax(tpr >= recall_90_precision)]           # Not shown\n",
    "# plt.plot([fpr_90, fpr_90], [0., recall_90_precision], \"r:\")   # Not shown\n",
    "# plt.plot([0.0, fpr_90], [recall_90_precision, recall_90_precision], \"r:\")  # Not shown\n",
    "# plt.plot([fpr_90], [recall_90_precision], \"ro\")               # Not shown\n",
    "# save_fig(\"roc_curve_plot\")                                    # Not shown\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5229491287230491"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_test, y_test_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8490640200461849"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_test, y_test_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5229491287230492"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "balanced_accuracy_score(y_test, y_test_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.18645418326693228"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision_score(y_test, y_test_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.10240700218818381"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_test, y_test_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}