ensemble_learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loading required library\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I will be using the credit card application dataset, where I will try to predict whether or not a given customer should be approved for their credit application."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b</td>\n",
       "      <td>30.83</td>\n",
       "      <td>0.000</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>w</td>\n",
       "      <td>v</td>\n",
       "      <td>1.25</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>1</td>\n",
       "      <td>f</td>\n",
       "      <td>g</td>\n",
       "      <td>202.0</td>\n",
       "      <td>0</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a</td>\n",
       "      <td>58.67</td>\n",
       "      <td>4.460</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>q</td>\n",
       "      <td>h</td>\n",
       "      <td>3.04</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>6</td>\n",
       "      <td>f</td>\n",
       "      <td>g</td>\n",
       "      <td>43.0</td>\n",
       "      <td>560</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a</td>\n",
       "      <td>24.50</td>\n",
       "      <td>0.500</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>q</td>\n",
       "      <td>h</td>\n",
       "      <td>1.50</td>\n",
       "      <td>t</td>\n",
       "      <td>f</td>\n",
       "      <td>0</td>\n",
       "      <td>f</td>\n",
       "      <td>g</td>\n",
       "      <td>280.0</td>\n",
       "      <td>824</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b</td>\n",
       "      <td>27.83</td>\n",
       "      <td>1.540</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>w</td>\n",
       "      <td>v</td>\n",
       "      <td>3.75</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>5</td>\n",
       "      <td>t</td>\n",
       "      <td>g</td>\n",
       "      <td>100.0</td>\n",
       "      <td>3</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>b</td>\n",
       "      <td>20.17</td>\n",
       "      <td>5.625</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>w</td>\n",
       "      <td>v</td>\n",
       "      <td>1.71</td>\n",
       "      <td>t</td>\n",
       "      <td>f</td>\n",
       "      <td>0</td>\n",
       "      <td>f</td>\n",
       "      <td>s</td>\n",
       "      <td>120.0</td>\n",
       "      <td>0</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  0      1      2  3  4  5  6     7  8  9   10 11 12     13   14 15\n",
       "0  b  30.83  0.000  u  g  w  v  1.25  t  t   1  f  g  202.0    0  +\n",
       "1  a  58.67  4.460  u  g  q  h  3.04  t  t   6  f  g   43.0  560  +\n",
       "2  a  24.50  0.500  u  g  q  h  1.50  t  f   0  f  g  280.0  824  +\n",
       "3  b  27.83  1.540  u  g  w  v  3.75  t  t   5  t  g  100.0    3  +\n",
       "4  b  20.17  5.625  u  g  w  v  1.71  t  f   0  f  s  120.0    0  +"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read the file \n",
    "\n",
    "cred_Data = pd.read_csv(\"credit_data.csv\", sep = \",\", header = None, na_values = \"?\")\n",
    "cred_Data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b</td>\n",
       "      <td>30.83</td>\n",
       "      <td>0.000</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>w</td>\n",
       "      <td>v</td>\n",
       "      <td>1.25</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>1</td>\n",
       "      <td>f</td>\n",
       "      <td>g</td>\n",
       "      <td>202.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a</td>\n",
       "      <td>58.67</td>\n",
       "      <td>4.460</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>q</td>\n",
       "      <td>h</td>\n",
       "      <td>3.04</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>6</td>\n",
       "      <td>f</td>\n",
       "      <td>g</td>\n",
       "      <td>43.0</td>\n",
       "      <td>560</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a</td>\n",
       "      <td>24.50</td>\n",
       "      <td>0.500</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>q</td>\n",
       "      <td>h</td>\n",
       "      <td>1.50</td>\n",
       "      <td>t</td>\n",
       "      <td>f</td>\n",
       "      <td>0</td>\n",
       "      <td>f</td>\n",
       "      <td>g</td>\n",
       "      <td>280.0</td>\n",
       "      <td>824</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b</td>\n",
       "      <td>27.83</td>\n",
       "      <td>1.540</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>w</td>\n",
       "      <td>v</td>\n",
       "      <td>3.75</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>5</td>\n",
       "      <td>t</td>\n",
       "      <td>g</td>\n",
       "      <td>100.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>b</td>\n",
       "      <td>20.17</td>\n",
       "      <td>5.625</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>w</td>\n",
       "      <td>v</td>\n",
       "      <td>1.71</td>\n",
       "      <td>t</td>\n",
       "      <td>f</td>\n",
       "      <td>0</td>\n",
       "      <td>f</td>\n",
       "      <td>s</td>\n",
       "      <td>120.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  0      1      2  3  4  5  6     7  8  9   10 11 12     13   14 15\n",
       "0  b  30.83  0.000  u  g  w  v  1.25  t  t   1  f  g  202.0    0  1\n",
       "1  a  58.67  4.460  u  g  q  h  3.04  t  t   6  f  g   43.0  560  1\n",
       "2  a  24.50  0.500  u  g  q  h  1.50  t  f   0  f  g  280.0  824  1\n",
       "3  b  27.83  1.540  u  g  w  v  3.75  t  t   5  t  g  100.0    3  1\n",
       "4  b  20.17  5.625  u  g  w  v  1.71  t  f   0  f  s  120.0    0  1"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# : + for approved and - for not approved\n",
    "# Changing the Classes to 1 & 0 from + & -\n",
    "\n",
    "cred_Data.loc[cred_Data[15] == '+' , 15] = 1\n",
    "cred_Data.loc[cred_Data[15] == '-' , 15] = 0\n",
    "cred_Data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     12\n",
       "1     12\n",
       "2      0\n",
       "3      6\n",
       "4      6\n",
       "5      9\n",
       "6      9\n",
       "7      0\n",
       "8      0\n",
       "9      0\n",
       "10     0\n",
       "11     0\n",
       "12     0\n",
       "13    13\n",
       "14     0\n",
       "15     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# finding the number of null values in each of the features \n",
    "\n",
    "cred_Data.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of raw data set  (690, 16)\n",
      "Data types of data set  0      object\n",
      "1     float64\n",
      "2     float64\n",
      "3      object\n",
      "4      object\n",
      "5      object\n",
      "6      object\n",
      "7     float64\n",
      "8      object\n",
      "9      object\n",
      "10      int64\n",
      "11     object\n",
      "12     object\n",
      "13    float64\n",
      "14      int64\n",
      "15     object\n",
      "dtype: object\n"
     ]
    }
   ],
   "source": [
    "# printing the shape and data types of each column\n",
    "\n",
    "print(\"Shape of raw data set \",cred_Data.shape)\n",
    "print(\"Data types of data set \",cred_Data.dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(653, 16)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Dropping all the rows with na values\n",
    "\n",
    "new_cred = cred_Data.dropna(axis = 0)\n",
    "new_cred.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     0\n",
       "1     0\n",
       "2     0\n",
       "3     0\n",
       "4     0\n",
       "5     0\n",
       "6     0\n",
       "7     0\n",
       "8     0\n",
       "9     0\n",
       "10    0\n",
       "11    0\n",
       "12    0\n",
       "13    0\n",
       "14    0\n",
       "15    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Verifying no null values exist\n",
    "\n",
    "new_cred.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Making dummy values from the categorical variables.\n",
    "\n",
    "# Seperating the categorical variables to make dummy variables\n",
    "\n",
    "cred_categorical = pd.get_dummies(new_cred[[0,3,4,5,6,8,9,11,12]])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Seperating the numerical variables\n",
    "\n",
    "cred_numerical = new_cred[[1,2,7,10,13,14]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(653, 46)\n",
      "(653,)\n",
      "0    1\n",
      "1    1\n",
      "2    1\n",
      "3    1\n",
      "4    1\n",
      "Name: 15, dtype: object\n"
     ]
    }
   ],
   "source": [
    "# Making the X variable which is a concatenation of categorical and numerical data\n",
    "\n",
    "X = pd.concat([cred_categorical,cred_numerical], axis = 1)\n",
    "print(X.shape)\n",
    "\n",
    "# target feature as y\n",
    "\n",
    "y = new_cred[15]\n",
    "print(y.shape)\n",
    "print(y.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0      1\n",
      "1      1\n",
      "2      1\n",
      "3      1\n",
      "4      1\n",
      "      ..\n",
      "685    0\n",
      "686    0\n",
      "687    0\n",
      "688    0\n",
      "689    0\n",
      "Name: 15, Length: 653, dtype: int32\n"
     ]
    }
   ],
   "source": [
    "# type of y is object, so sklearn cannot recognize its type.\n",
    "# changing type of y as integer\n",
    "\n",
    "y = y.astype(\"int\")\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>36</th>\n",
       "      <th>37</th>\n",
       "      <th>38</th>\n",
       "      <th>39</th>\n",
       "      <th>40</th>\n",
       "      <th>41</th>\n",
       "      <th>42</th>\n",
       "      <th>43</th>\n",
       "      <th>44</th>\n",
       "      <th>45</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.271111</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.043860</td>\n",
       "      <td>0.014925</td>\n",
       "      <td>0.1010</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.713016</td>\n",
       "      <td>0.159286</td>\n",
       "      <td>0.106667</td>\n",
       "      <td>0.089552</td>\n",
       "      <td>0.0215</td>\n",
       "      <td>0.00560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.170635</td>\n",
       "      <td>0.017857</td>\n",
       "      <td>0.052632</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.1400</td>\n",
       "      <td>0.00824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.223492</td>\n",
       "      <td>0.055000</td>\n",
       "      <td>0.131579</td>\n",
       "      <td>0.074627</td>\n",
       "      <td>0.0500</td>\n",
       "      <td>0.00003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.101905</td>\n",
       "      <td>0.200893</td>\n",
       "      <td>0.060000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0600</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 46 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    0    1    2    3    4    5    6    7    8    9   ...   36   37   38   39  \\\n",
       "0  0.0  1.0  0.0  1.0  0.0  1.0  0.0  0.0  0.0  0.0  ...  0.0  1.0  0.0  0.0   \n",
       "1  1.0  0.0  0.0  1.0  0.0  1.0  0.0  0.0  0.0  0.0  ...  0.0  1.0  0.0  0.0   \n",
       "2  1.0  0.0  0.0  1.0  0.0  1.0  0.0  0.0  0.0  0.0  ...  0.0  1.0  0.0  0.0   \n",
       "3  0.0  1.0  0.0  1.0  0.0  1.0  0.0  0.0  0.0  0.0  ...  1.0  1.0  0.0  0.0   \n",
       "4  0.0  1.0  0.0  1.0  0.0  1.0  0.0  0.0  0.0  0.0  ...  0.0  0.0  0.0  1.0   \n",
       "\n",
       "         40        41        42        43      44       45  \n",
       "0  0.271111  0.000000  0.043860  0.014925  0.1010  0.00000  \n",
       "1  0.713016  0.159286  0.106667  0.089552  0.0215  0.00560  \n",
       "2  0.170635  0.017857  0.052632  0.000000  0.1400  0.00824  \n",
       "3  0.223492  0.055000  0.131579  0.074627  0.0500  0.00003  \n",
       "4  0.101905  0.200893  0.060000  0.000000  0.0600  0.00000  \n",
       "\n",
       "[5 rows x 46 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Normalising the data sets\n",
    "# Import library function\n",
    "\n",
    "from sklearn import preprocessing\n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "\n",
    "# Transforming with the scaler function\n",
    "X_scaled = pd.DataFrame(min_max_scaler.fit_transform(X))\n",
    "\n",
    "# Printing the output\n",
    "X_scaled.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Splitting the data into train and test sets\n",
    "# Importing library function\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.3, random_state=123)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression()"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Defining the LogisticRegression function\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "log_reg_model = LogisticRegression()\n",
    "\n",
    "# Fitting a logistic regression model on the training set\n",
    "log_reg_model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of Logistic regression model prediction on test set: 0.89\n"
     ]
    }
   ],
   "source": [
    "# predictions on the test set\n",
    "\n",
    "pred = log_reg_model.predict(X_test)\n",
    "print(\"Accuracy of Logistic regression model prediction on test set: {:.2f}\".format(log_reg_model.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[93 14]\n",
      " [ 8 81]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.87      0.89       107\n",
      "           1       0.85      0.91      0.88        89\n",
      "\n",
      "    accuracy                           0.89       196\n",
      "   macro avg       0.89      0.89      0.89       196\n",
      "weighted avg       0.89      0.89      0.89       196\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "# Confusion Matrix for the model\n",
    "print(confusion_matrix(y_test, pred))\n",
    "\n",
    "# Classification report for the model\n",
    "print(classification_report(y_test, pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### We can see from the output, we achieved an accuracy level of 0.89 with the logistic regression model. we will try different methods of ensemble learning with simple methods"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ensemble Model Using the Averaging Technique:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The averaging method creates an ensemble by combining the predictions of base learners and averaging the prediction probabilities."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining the three base models. \n",
    "\n",
    "model1 = LogisticRegression(random_state = 20)\n",
    "model2 = KNeighborsClassifier(n_neighbors = 5)\n",
    "model3 = RandomForestClassifier(n_estimators = 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(n_estimators=500)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fitting all three models on the training data\n",
    "\n",
    "model1.fit(X_train, y_train)\n",
    "model2.fit(X_train, y_train)\n",
    "model3.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# predicting the probabilities of each model on the test set\n",
    "\n",
    "pred1 = model1.predict_proba(X_test)\n",
    "pred2 = model2.predict_proba(X_test)\n",
    "pred3 = model3.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculating the ensemble prediction by averaging three base model predictions\n",
    "\n",
    "ensemble_pred=(pred1 + pred2 + pred3)/3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.90516917, 0.09483083],\n",
       "       [0.96396145, 0.03603855],\n",
       "       [0.18357473, 0.81642527],\n",
       "       [0.05083488, 0.94916512],\n",
       "       [0.85873263, 0.14126737],\n",
       "       [0.01209282, 0.98790718],\n",
       "       [0.18589271, 0.81410729],\n",
       "       [0.10031852, 0.89968148],\n",
       "       [0.39451137, 0.60548863]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Displaying first 10 rows of the ensemble predictions\n",
    "\n",
    "ensemble_pred[0:9,:]\n",
    "\n",
    "# we have two probabilities for each example corresponding to each class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 1]\n",
      "[0 1]\n",
      "[0 1]\n"
     ]
    }
   ],
   "source": [
    "#prediction output has two columns corresponding to each class - Printing the order of classes for each model\n",
    "\n",
    "print(model1.classes_)\n",
    "print(model2.classes_)\n",
    "print(model3.classes_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0,\n",
       "       0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0,\n",
       "       1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0,\n",
       "       0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0,\n",
       "       0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0,\n",
       "       1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1,\n",
       "       0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1,\n",
       "       1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0,\n",
       "       1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# getting the final predictions for each example from the output probabilities.\n",
    "# The final prediction will be the class with the highest probability\n",
    "\n",
    "pred = np.argmax(ensemble_pred, axis = 1)\n",
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[96 11]\n",
      " [ 8 81]]\n"
     ]
    }
   ],
   "source": [
    "# generating confusion matrix\n",
    "\n",
    "confusion_Matrix = confusion_matrix(y_test, pred)\n",
    "print(confusion_Matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.90      0.91       107\n",
      "           1       0.88      0.91      0.90        89\n",
      "\n",
      "    accuracy                           0.90       196\n",
      "   macro avg       0.90      0.90      0.90       196\n",
      "weighted avg       0.90      0.90      0.90       196\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# generating classification report\n",
    "\n",
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_test, pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### It is implemented the averaging technique for ensemble learning. As you can see from the classification report, we managed to improve the accuracy rate from 0.89 to 0.90 by averaging method. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ensemble Model Using the Weighted Averaging Technique"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The weighted averaging method was similar to the averaging method.The difference was in the way the predictions were combined. In this method, arbitrary weights were applied to the predictions of individual learners to get the final prediction."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#In the weighted averaging method, weights are assigned arbitrarily based on our judgment of each of the predictions\n",
    "# weights are assigned  that the sum of all weights becomes 1 \n",
    "# Calculating the ensemble prediction by applying weights for each prediction\n",
    "\n",
    "ensemble_pred = (pred1 * 0.60 + pred2 * 0.20 + pred3 * 0.20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.9229045 , 0.0770955 ],\n",
       "       [0.9495306 , 0.0504694 ],\n",
       "       [0.14323452, 0.85676548],\n",
       "       [0.08830278, 0.91169722],\n",
       "       [0.88651874, 0.11348126],\n",
       "       [0.01936707, 0.98063293],\n",
       "       [0.13940688, 0.86059312],\n",
       "       [0.09897334, 0.90102666],\n",
       "       [0.44372046, 0.55627954]])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Displaying first 10 rows of the ensemble predictions\n",
    "ensemble_pred[0:9,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 1]\n",
      "[0 1]\n",
      "[0 1]\n"
     ]
    }
   ],
   "source": [
    "# Printing the order of classes for each model\n",
    "print(model1.classes_)\n",
    "print(model2.classes_)\n",
    "print(model3.classes_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0,\n",
       "       0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0,\n",
       "       1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0,\n",
       "       0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0,\n",
       "       0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0,\n",
       "       1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1,\n",
       "       0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1,\n",
       "       1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0,\n",
       "       1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculating the final predictions from the probabilities\n",
    "# getting the final predictions for each example from the output probabilities using the np.argmax() function\n",
    "\n",
    "pred = np.argmax(ensemble_pred, axis =1)\n",
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[94 13]\n",
      " [ 8 81]]\n"
     ]
    }
   ],
   "source": [
    "# Generating confusion matrix\n",
    "\n",
    "confusionMatrix = confusion_matrix(y_test, pred)\n",
    "print(confusionMatrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.88      0.90       107\n",
      "           1       0.86      0.91      0.89        89\n",
      "\n",
      "    accuracy                           0.89       196\n",
      "   macro avg       0.89      0.89      0.89       196\n",
      "weighted avg       0.89      0.89      0.89       196\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Generating classification report\n",
    "\n",
    "print(classification_report(y_test, pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Different Weights by Weighted Averaging Technique\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculating the ensemble prediction by applying weights for each prediction\n",
    "\n",
    "ensemble_pred=(pred1 *0.70 + pred2 * 0.15  +pred3 * 0.15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generating predictions from probabilities\n",
    "pred = np.argmax(ensemble_pred,axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[94 13]\n",
      " [ 7 82]]\n"
     ]
    }
   ],
   "source": [
    "# Generating confusion matrix\n",
    "confusionMatrix = confusion_matrix(y_test, pred)\n",
    "print(confusionMatrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.88      0.90       107\n",
      "           1       0.86      0.92      0.89        89\n",
      "\n",
      "    accuracy                           0.90       196\n",
      "   macro avg       0.90      0.90      0.90       196\n",
      "weighted avg       0.90      0.90      0.90       196\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Generating classification report\n",
    "\n",
    "print(classification_report(y_test, pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is  implemented the weighted averaging technique for ensemble learning. We did two iterations with the weights. We saw that in the second iteration, where we increased the weight of the logistic regression prediction from 0.6 to 0.7, the accuracy actually improved from 0.89 to 0.90. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Max Voting Method"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Max voting was a technique that arrived at final predictions based on the votes of the majority of the base learners."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# models\n",
    "model1 = LogisticRegression(random_state= 20)\n",
    "model2 = KNeighborsClassifier(n_neighbors= 6)\n",
    "model3 = RandomForestClassifier(n_estimators=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining the ensemble model using VotingClassifier\n",
    "# voting is \"hard, \" which means that the output will be class labels and not probabilities\n",
    "model = VotingClassifier(estimators=[( 'lr', model1),( 'knn', model2),( 'rf',model3)], voting= 'hard')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VotingClassifier(estimators=[('lr', LogisticRegression(random_state=20)),\n",
       "                             ('knn', KNeighborsClassifier(n_neighbors=6)),\n",
       "                             ('rf', RandomForestClassifier(n_estimators=500))])"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fitting the model on the training set\n",
    "\n",
    "model.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9081632653061225"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# predicting accuracy score on the test set\n",
    "\n",
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generating the predictions on the test set\n",
    "\n",
    "preds = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[97 10]\n",
      " [ 8 81]]\n"
     ]
    }
   ],
   "source": [
    "# Confusion matrix for the test set\n",
    "\n",
    "print(confusion_matrix(y_test, preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.91      0.92       107\n",
      "           1       0.89      0.91      0.90        89\n",
      "\n",
      "    accuracy                           0.91       196\n",
      "   macro avg       0.91      0.91      0.91       196\n",
      "weighted avg       0.91      0.91      0.91       196\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# classification report for Max Voting Technique\n",
    "\n",
    "print(classification_report(y_test, preds))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is implemented the max voting technique for ensemble learning. As you can see from the classification report, the results are better than averaging method .This result is more balanced. The recall value for worthy customers is high, at 91 % for their credit application."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Advanced Techniques for Ensemble Learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I will implement some advanced techniques for ensemble learning such as Bagging, Boosting and Stacking/blending."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ensemble Learning with Bagging Method"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Bagging : Multiple subsets of the data are created using bootstrapping. On each of these subsets of data, a base learner is fitted and the predictions generated. These predictions from all the base learners are then averaged to get the meta learner or the final predictions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining the base learner\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "random_forest_model = RandomForestClassifier(random_state = 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating the bagging meta learner\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "bagging_learner = BaggingClassifier(base_estimator = random_forest_model, n_estimators= 10, max_samples=0.6, max_features= 0.6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fitting the model using the meta learner\n",
    "\n",
    "model = bagging_learner.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predicting on the test set using the model\n",
    "\n",
    "pred = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[99  8]\n",
      " [12 77]]\n"
     ]
    }
   ],
   "source": [
    "# Printing the confusion matrix\n",
    "# from sklearn.metrics import confusion_matrix\n",
    "\n",
    "print(confusion_matrix(y_test, pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      0.93      0.91       107\n",
      "           1       0.91      0.87      0.89        89\n",
      "\n",
      "    accuracy                           0.90       196\n",
      "   macro avg       0.90      0.90      0.90       196\n",
      "weighted avg       0.90      0.90      0.90       196\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Printing the classification report\n",
    "\n",
    "print(classification_report(y_test, pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is implemented ensemble learning using bagging method. As you can see from the classification report, we have a slightly lower result (0.90) than what we got from the max voting methods (0.91). However, looking at the results,\n",
    "We can  see that  the recall value for identifying unworthy customers is around 93%, which means that only around 7% of the unworthy customers are wrongly classified as creditworthy."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ensemble Learning with Boosting Method"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Boosting methods works in a sequential manner. It works on the principle of correcting the prediction errors of each base learner. The base learners are fit sequentially one after the other. A base learner tries to correct the error generated by the previous learner and this process continues until a superior meta learner is created. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining the base learner\n",
    "\n",
    "model_logistic_reg = LogisticRegression(random_state = 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining the boosting meta learner\n",
    "\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "\n",
    "boosting_model = AdaBoostClassifier(base_estimator = model_logistic_reg, n_estimators= 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fitting the model on the training set\n",
    "\n",
    "fitted_model = boosting_model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Getting the predictions from the boosting model\n",
    "\n",
    "pred = fitted_model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[96 11]\n",
      " [ 8 81]]\n"
     ]
    }
   ],
   "source": [
    "# generating the confusion matrix\n",
    "\n",
    "print(confusion_matrix(y_test, pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.90      0.91       107\n",
      "           1       0.88      0.91      0.90        89\n",
      "\n",
      "    accuracy                           0.90       196\n",
      "   macro avg       0.90      0.90      0.90       196\n",
      "weighted avg       0.90      0.90      0.90       196\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# generating confusion matrix\n",
    "\n",
    "print(classification_report(y_test, pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is implemented the ensemble learning model using boosting. As you can see from the classification report, we got very similar results (0.90) to the bagging method.The results are more balanced compared to result of the bagging method.(the recall values of 0.93 and 0.87 in bagging method). Here, we can see that the recall value of the unworthy customers (90%) and that of creditworthy customers (91%) are quite close to each other, which indicates a very balanced result."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ensemble Learning with Stacking Method"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Stacking is an ensemble learning technique that combines multiple classification or regression models via a meta-classifier or a meta-regressor. The base level models are trained based on a complete training set, then the meta-model is trained on the outputs of the base level model as features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing the meta learner (logistic regression) and base learners (knn and random forest)\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "base_learner_knn= KNeighborsClassifier(n_neighbors = 5)\n",
    "base_learner_random_forest = RandomForestClassifier(random_state = 50)\n",
    "meta_learner_log_reg = LogisticRegression(random_state = 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating the StackingClassifier\n",
    "# !pip install mlxtend \n",
    "from mlxtend.classifier import StackingClassifier\n",
    "\n",
    "stackclf = StackingClassifier(classifiers=[base_learner_knn,base_learner_random_forest],\n",
    "meta_classifier= meta_learner_log_reg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fitting the model on the training set\n",
    "\n",
    "main_model = stackclf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generating predictions on test set\n",
    "\n",
    "pred = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[99  8]\n",
      " [12 77]]\n"
     ]
    }
   ],
   "source": [
    "# Generating the confusion matrix\n",
    "\n",
    "print(confusion_matrix(y_test, pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      0.93      0.91       107\n",
      "           1       0.91      0.87      0.89        89\n",
      "\n",
      "    accuracy                           0.90       196\n",
      "   macro avg       0.90      0.90      0.90       196\n",
      "weighted avg       0.90      0.90      0.90       196\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Generating the classification report\n",
    "\n",
    "print(classification_report(y_test, pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is implemented the ensemble learning model using stacking. As you can see from the classification report, we did  get some improvements over the main model using the stacking classifier."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Comparison of Advanced Ensemble Techniques and Picking the best Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Initially for Bagging Method, I will define base learner and  meta learner, in the second part fitting the model, in the third part generating predictions and confusion matrix, in the last part printing classification report "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[93 14]\n",
      " [ 7 82]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.87      0.90       107\n",
      "           1       0.85      0.92      0.89        89\n",
      "\n",
      "    accuracy                           0.89       196\n",
      "   macro avg       0.89      0.90      0.89       196\n",
      "weighted avg       0.90      0.89      0.89       196\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Bagging Method\n",
    "from sklearn.ensemble import BaggingClassifier \n",
    "from sklearn.linear_model import LogisticRegression \n",
    "from sklearn.metrics import confusion_matrix \n",
    "from sklearn.metrics import classification_report \n",
    "\n",
    "# Creating the bagging meta learner\n",
    "base_lerner_log_reg = LogisticRegression(random_state = 50) \n",
    "bagging_learner = BaggingClassifier(base_estimator = base_lerner_log_reg, n_estimators= 15, max_samples = 0.8, max_features = 0.8) \n",
    "\n",
    "# Fitting the model using the meta learner \n",
    "model = bagging_learner.fit(X_train, y_train) \n",
    "\n",
    "# Predicting on the test set using the model \n",
    "pred = model.predict(X_test) \n",
    "\n",
    "print(confusion_matrix(y_test, pred)) \n",
    "\n",
    "print(classification_report(y_test, pred)) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Output for Bagging Method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[96 11]\n",
      " [ 8 81]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.90      0.91       107\n",
      "           1       0.88      0.91      0.90        89\n",
      "\n",
      "    accuracy                           0.90       196\n",
      "   macro avg       0.90      0.90      0.90       196\n",
      "weighted avg       0.90      0.90      0.90       196\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Boosting Method\n",
    "\n",
    "from sklearn.ensemble import BaggingClassifier \n",
    "from sklearn.ensemble import RandomForestClassifier \n",
    "from sklearn.metrics import confusion_matrix \n",
    "from sklearn.metrics import classification_report  \n",
    "\n",
    "base_learner_random_forest = RandomForestClassifier(random_state = 50) # base learner\n",
    "boosting = AdaBoostClassifier(base_estimator = base_learner_random_forest, n_estimators = 100) # meta learner\n",
    "model = boosting.fit(X_train, y_train) \n",
    "pred = model.predict(X_test) \n",
    "print(confusion_matrix(y_test, pred)) \n",
    "print(classification_report(y_test, pred)) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Output for Boosting Method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[94 13]\n",
      " [ 8 81]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.88      0.90       107\n",
      "           1       0.86      0.91      0.89        89\n",
      "\n",
      "    accuracy                           0.89       196\n",
      "   macro avg       0.89      0.89      0.89       196\n",
      "weighted avg       0.89      0.89      0.89       196\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Importing the meta learner and base learners \n",
    "from sklearn.linear_model import LogisticRegression \n",
    "from sklearn.neighbors import KNeighborsClassifier \n",
    "from sklearn.ensemble import RandomForestClassifier \n",
    "from mlxtend.classifier import StackingClassifier \n",
    "from sklearn.metrics import confusion_matrix \n",
    "from sklearn.metrics import classification_report \n",
    "\n",
    "base_learner_knn= KNeighborsClassifier(n_neighbors = 5)\n",
    "base_learner_log_reg= RandomForestClassifier(random_state = 50)\n",
    "meta_learner_random_forest = LogisticRegression(random_state = 50)\n",
    "\n",
    "stack_clf = StackingClassifier(classifiers = [base_learner_knn, base_learner_log_reg,], meta_classifier= meta_learner_random_forest) \n",
    "\n",
    "model = stack_clf.fit(X_train, y_train) \n",
    "pred = model.predict(X_test) \n",
    "\n",
    "print(confusion_matrix(y_test, pred)) \n",
    "print(classification_report(y_test, pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Output for Stacking Method"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I implemented on the credit card dataset with  three advanced ensemble techniques.(bagging, boosting and stacking) Based on the metrics, It is founded that boosting (0.90) has better results than bagging (0.89) and stacking (0.89). Therefore, select the boosting algorithm to boost the performance of our models because boosting algorithm generated more balanced results where the recall value of both creditworthy and not creditworthy customers (90% and 91%) is similar."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}