Beijing_PM_Stack-CV.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "from sklearn.svm import SVR, LinearSVR\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.linear_model import LinearRegression, Lasso"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = pd.read_csv('C:\\\\beijing_engineered.csv')\n",
    "df = pd.DataFrame(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pm25</th>\n",
       "      <th>dewp</th>\n",
       "      <th>temp</th>\n",
       "      <th>cws</th>\n",
       "      <th>target</th>\n",
       "      <th>cbwd_1</th>\n",
       "      <th>cbwd_2</th>\n",
       "      <th>cbwd_3</th>\n",
       "      <th>clunar</th>\n",
       "      <th>cstorm</th>\n",
       "      <th>...</th>\n",
       "      <th>chour_16.0</th>\n",
       "      <th>chour_17.0</th>\n",
       "      <th>chour_18.0</th>\n",
       "      <th>chour_19.0</th>\n",
       "      <th>chour_20.0</th>\n",
       "      <th>chour_21.0</th>\n",
       "      <th>chour_22.0</th>\n",
       "      <th>chour_23.0</th>\n",
       "      <th>pm25_1</th>\n",
       "      <th>pm25_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>159</td>\n",
       "      <td>-11</td>\n",
       "      <td>-5</td>\n",
       "      <td>3</td>\n",
       "      <td>181</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>148</td>\n",
       "      <td>129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>181</td>\n",
       "      <td>-7</td>\n",
       "      <td>-5</td>\n",
       "      <td>5</td>\n",
       "      <td>138</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>159</td>\n",
       "      <td>148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>138</td>\n",
       "      <td>-7</td>\n",
       "      <td>-5</td>\n",
       "      <td>6</td>\n",
       "      <td>109</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>181</td>\n",
       "      <td>159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>109</td>\n",
       "      <td>-7</td>\n",
       "      <td>-6</td>\n",
       "      <td>7</td>\n",
       "      <td>105</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>138</td>\n",
       "      <td>181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>105</td>\n",
       "      <td>-7</td>\n",
       "      <td>-6</td>\n",
       "      <td>8</td>\n",
       "      <td>124</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>109</td>\n",
       "      <td>138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43792</th>\n",
       "      <td>10</td>\n",
       "      <td>-22</td>\n",
       "      <td>-2</td>\n",
       "      <td>226</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43793</th>\n",
       "      <td>8</td>\n",
       "      <td>-23</td>\n",
       "      <td>-2</td>\n",
       "      <td>231</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43794</th>\n",
       "      <td>10</td>\n",
       "      <td>-22</td>\n",
       "      <td>-3</td>\n",
       "      <td>237</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43795</th>\n",
       "      <td>10</td>\n",
       "      <td>-22</td>\n",
       "      <td>-3</td>\n",
       "      <td>242</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43796</th>\n",
       "      <td>8</td>\n",
       "      <td>-22</td>\n",
       "      <td>-4</td>\n",
       "      <td>246</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>43797 rows × 49 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       pm25  dewp  temp  cws  target  cbwd_1  cbwd_2  cbwd_3  clunar  cstorm  \\\n",
       "0       159   -11    -5    3     181       0       0       0       0       0   \n",
       "1       181    -7    -5    5     138       0       0       0       0       0   \n",
       "2       138    -7    -5    6     109       0       0       0       0       0   \n",
       "3       109    -7    -6    7     105       0       0       0       0       0   \n",
       "4       105    -7    -6    8     124       0       0       0       0       0   \n",
       "...     ...   ...   ...  ...     ...     ...     ...     ...     ...     ...   \n",
       "43792    10   -22    -2  226       8       0       1       0       0       0   \n",
       "43793     8   -23    -2  231      10       0       1       0       0       0   \n",
       "43794    10   -22    -3  237      10       0       1       0       0       0   \n",
       "43795    10   -22    -3  242       8       0       1       0       0       0   \n",
       "43796     8   -22    -4  246      12       0       1       0       0       0   \n",
       "\n",
       "       ...  chour_16.0  chour_17.0 chour_18.0  chour_19.0  chour_20.0  \\\n",
       "0      ...           0           0          0           0           0   \n",
       "1      ...           0           0          0           0           0   \n",
       "2      ...           0           0          0           0           0   \n",
       "3      ...           0           0          0           0           0   \n",
       "4      ...           0           0          0           0           0   \n",
       "...    ...         ...         ...        ...         ...         ...   \n",
       "43792  ...           0           0          0           1           0   \n",
       "43793  ...           0           0          0           0           1   \n",
       "43794  ...           0           0          0           0           0   \n",
       "43795  ...           0           0          0           0           0   \n",
       "43796  ...           0           0          0           0           0   \n",
       "\n",
       "       chour_21.0  chour_22.0  chour_23.0  pm25_1  pm25_2  \n",
       "0               0           0           0     148     129  \n",
       "1               0           0           0     159     148  \n",
       "2               0           0           0     181     159  \n",
       "3               0           0           0     138     181  \n",
       "4               0           0           0     109     138  \n",
       "...           ...         ...         ...     ...     ...  \n",
       "43792           0           0           0       9       8  \n",
       "43793           0           0           0      10       9  \n",
       "43794           1           0           0       8      10  \n",
       "43795           0           1           0      10       8  \n",
       "43796           0           0           1      10      10  \n",
       "\n",
       "[43797 rows x 49 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['date'] = pd.to_datetime(df.date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setting DateTime index\n",
    "df['date'] = pd.to_datetime(df.date)\n",
    "df.set_index('date', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pm25</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>43797.000000</td>\n",
       "      <td>43797.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>98.025367</td>\n",
       "      <td>98.022011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>90.522403</td>\n",
       "      <td>90.522867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>74.000000</td>\n",
       "      <td>74.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>134.000000</td>\n",
       "      <td>134.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>994.000000</td>\n",
       "      <td>994.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               pm25        target\n",
       "count  43797.000000  43797.000000\n",
       "mean      98.025367     98.022011\n",
       "std       90.522403     90.522867\n",
       "min        0.000000      0.000000\n",
       "25%       30.000000     30.000000\n",
       "50%       74.000000     74.000000\n",
       "75%      134.000000    134.000000\n",
       "max      994.000000    994.000000"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['pm25', 'target']].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Subsetting Data for Cross-Validation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The cross-validation procedure requires that the train-validation data (ex-holdout test set) be subsetted into **five time-ordered portions** for CV=5, using the **forward chain approach**.\n",
    "\n",
    "Each of the five folds are then split into **three time-ordered portions** of which the final portion is the **validation set**, which is of a **fixed length**."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Exclude the last 10% of observations, which is the holdout-test set that will not be used in the CV\n",
    "X = df.iloc[:-4380]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39417"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Size of overall CV train-validation data\n",
    "len(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Each fold** of the 5-folds of data are then **split 3-ways**:\n",
    "- A **fixed validation set** sized at 10% of the overall CV data\n",
    "- 2/3 of remaining fold for the base training data (earliest in time)\n",
    "- 1/3 of remaining fold for the meta training data (later in time)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Given **CV=5**, that means 5 X validation sample lengths should be set ahead of the first training data (base+meta) fold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3942"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Each fixed validation sub-sample size\n",
    "n_valid = round(len(X)*0.1)\n",
    "n_valid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_split = 0\n",
    "splits_list = []\n",
    "\n",
    "while n_split < len(X):\n",
    "    if n_split == 0:\n",
    "        n_split = len(X) - (n_valid*5) # At the start, set the first training fold without the validation set\n",
    "    \n",
    "    else:\n",
    "        n_split += n_valid # Add the fixed validation set to each training fold\n",
    "        splits_list.append(n_split)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[23649, 27591, 31533, 35475, 39417]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Index numbers for where each CV fold's splice ends\n",
    "splits_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generating DataFrames of the Rolling Forward-Chained Folds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_list = [] # List of the CV folds as DataFrames\n",
    "\n",
    "for i in splits_list:\n",
    "    df_list.append(X[:i])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "23649"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_list[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Confirming that the final split covers the full CV data\n",
    "len(df_list[4]) - len(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Post-Gridsearch Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lasso Regression\n",
    "las = Lasso(fit_intercept=1, alpha=0.05, max_iter=10000, random_state=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Multi-Layer Perceptron\n",
    "mlp = MLPRegressor(hidden_layer_sizes=(36,12), alpha=1e-05, activation='relu', early_stopping=True,\n",
    "                   max_iter=20000, random_state=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Linear Support Vector Regression\n",
    "svr = LinearSVR(epsilon=11, C=37, fit_intercept=True, loss='squared_epsilon_insensitive', \n",
    "                max_iter=30000, random_state=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Support Vector Machine\n",
    "svm = SVR(kernel='poly', gamma=0.2, C=0.1, max_iter=30000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Random Forest Regressor\n",
    "rf = RandomForestRegressor(n_estimators=375, min_samples_split=2, max_features='sqrt', random_state=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# XG Boost Regressor\n",
    "xgb = xgb.XGBRegressor(n_estimators=100, gamma=230, eta=0.1, subsample=0.75, objective='reg:squarederror', \n",
    "                       random_state=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dictionary of above base models\n",
    "base_dict = {\n",
    "    'Lasso Regression': las,\n",
    "    'Multi-Layer Perceptron': mlp, \n",
    "    'Support Vector Regression': svr,\n",
    "    'Support Vector Machine': svm,\n",
    "    'Random Forest': rf,\n",
    "    'XG Boost': xgb\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setting Linear Regression as the final stage meta model\n",
    "meta = LinearRegression()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = StandardScaler()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fold_scoring(df_fold, meta_model, base_dict):\n",
    "    '''\n",
    "    Cycle through a dictionary of base models, fitting them then making predictions. The predictions are also \n",
    "    used as inputs for the meta model. All model predictions on the validation set are scored according to \n",
    "    MAE & RMSE. The output is two DataFrames containing the all-model scores.\n",
    "    \n",
    "    df_fold: DataFrame containing the sub-sample fold of interest, with the target variable in the last column\n",
    "    \n",
    "    meta_model: Meta model\n",
    "    \n",
    "    base_dict: Dictionary of base models\n",
    "    '''    \n",
    "    \n",
    "    # Empty scorer DataFrames\n",
    "    df_mae = pd.DataFrame(columns=base_dict.keys()) # Columns of DF will accord with base_dict keys\n",
    "    df_rmse = pd.DataFrame(columns=base_dict.keys())\n",
    "    \n",
    "    # Empty dictionaries to hold the base model predictions for the meta model\n",
    "    meta_dict1 = {}\n",
    "    meta_dict2 = {}\n",
    "    \n",
    "    # Split the data length into 2/3 base-training data and 1/3 meta-training data\n",
    "    length = len(df_fold) - n_valid # n_valid is given as per above\n",
    "    n_meta = round(length/3)\n",
    "    n_base = n_meta * 2\n",
    "    \n",
    "    X_base = df_fold.iloc[:n_base,:]\n",
    "    X_meta = df_fold.iloc[n_base:(n_base + n_meta),:]\n",
    "    X_valid = df_fold.iloc[length:(length + n_valid),:]\n",
    "    \n",
    "    # Extract target variables from each sub-sample\n",
    "    y_meta = X_meta.pop('target')\n",
    "    y_base = X_base.pop('target')\n",
    "    y_valid = X_valid.pop('target')\n",
    "    \n",
    "    # Scaling the sub-samples\n",
    "    X_bsc = scaler.fit_transform(X_base)\n",
    "    X_msc = scaler.transform(X_meta)\n",
    "    X_vsc = scaler.transform(X_valid)\n",
    "    \n",
    "    # Fit the base models to the base-training set, and generate predictions\n",
    "    for key, reg in base_dict.items():\n",
    "        reg.fit(X_bsc, y_base)\n",
    "        \n",
    "        z_meta = reg.predict(X_msc).tolist() # Generate predictions on meta-training set\n",
    "        meta_dict1[key] = z_meta # Append predictions to dictionary\n",
    "        \n",
    "        z_valid = reg.predict(X_vsc).tolist() # Generate predictions on validation set for meta model\n",
    "        meta_dict2[key] = z_valid\n",
    "\n",
    "        df_mae[key] = pd.Series(round(mean_absolute_error(z_valid, y_valid), 4)) # Generate MAE scores for base models\n",
    "        df_rmse[key] = pd.Series(round(mean_squared_error(z_valid, y_valid, squared=False), 4)) # Generate RMSE scores\n",
    "    \n",
    "    # Transform dictionary of predictions for meta model into DataFrames        \n",
    "    df_meta1 = pd.DataFrame.from_dict(meta_dict1)\n",
    "    df_meta2 = pd.DataFrame.from_dict(meta_dict2)\n",
    "    \n",
    "    # Train meta model using base models' predictions of meta-training set\n",
    "    meta_model.fit(df_meta1, y_meta)\n",
    "    \n",
    "    # Generate meta model predictions of validation set\n",
    "    meta_y = meta_model.predict(df_meta2)            \n",
    "    \n",
    "    # Obtain scores from meta model, and insert into respective scorer DataFrames\n",
    "    df_mae['Stack Model'] = round(mean_absolute_error(meta_y, y_valid), 4)\n",
    "    df_rmse['Stack Model'] = round(mean_squared_error(meta_y, y_valid, squared=False), 4)\n",
    "\n",
    "    return df_mae, df_rmse\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stack_scores(df1, df2, df3, df4, df5):\n",
    "    '''\n",
    "    Objective: Stack the scorer DataFrames, plus calculate mean, median and standard deviation of scores\n",
    "\n",
    "    '''    \n",
    "    new_index = ['1st fold', '2nd fold', '3rd fold', '4th fold', '5th fold']\n",
    "    \n",
    "    df_new = pd.concat([df1, df2, df3, df4, df5])\n",
    "    df_new.index = new_index\n",
    "    \n",
    "    x = len(df_new)\n",
    "    \n",
    "    df_new.loc['mean'] = round(df_new.mean(), 4)\n",
    "    df_new.loc['median'] = round(df_new.iloc[:x,:].median(), 4)\n",
    "    df_new.loc['stdev'] = round(df_new.iloc[:x,:].std(), 4)\n",
    "    \n",
    "    return df_new\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scoring Each Fold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Tan\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:985: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\Tan\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:255: ConvergenceWarning: Solver terminated early (max_iter=30000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  warnings.warn('Solver terminated early (max_iter=%i).'\n"
     ]
    }
   ],
   "source": [
    "mae_1st, rmse_1st = fold_scoring(df_list[0], meta, base_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lasso Regression</th>\n",
       "      <th>Multi-Layer Perceptron</th>\n",
       "      <th>Support Vector Regression</th>\n",
       "      <th>Support Vector Machine</th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>XG Boost</th>\n",
       "      <th>Stack Model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11.2777</td>\n",
       "      <td>12.5906</td>\n",
       "      <td>12.2504</td>\n",
       "      <td>13.483</td>\n",
       "      <td>13.2545</td>\n",
       "      <td>11.4165</td>\n",
       "      <td>11.0952</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Lasso Regression  Multi-Layer Perceptron  Support Vector Regression  \\\n",
       "0           11.2777                 12.5906                    12.2504   \n",
       "\n",
       "   Support Vector Machine  Random Forest  XG Boost  Stack Model  \n",
       "0                  13.483        13.2545   11.4165      11.0952  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mae_1st"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lasso Regression</th>\n",
       "      <th>Multi-Layer Perceptron</th>\n",
       "      <th>Support Vector Regression</th>\n",
       "      <th>Support Vector Machine</th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>XG Boost</th>\n",
       "      <th>Stack Model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>18.9254</td>\n",
       "      <td>19.7555</td>\n",
       "      <td>19.4904</td>\n",
       "      <td>20.9506</td>\n",
       "      <td>19.9403</td>\n",
       "      <td>18.7335</td>\n",
       "      <td>18.5377</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Lasso Regression  Multi-Layer Perceptron  Support Vector Regression  \\\n",
       "0           18.9254                 19.7555                    19.4904   \n",
       "\n",
       "   Support Vector Machine  Random Forest  XG Boost  Stack Model  \n",
       "0                 20.9506        19.9403   18.7335      18.5377  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rmse_1st"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Tan\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:985: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\Tan\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:255: ConvergenceWarning: Solver terminated early (max_iter=30000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  warnings.warn('Solver terminated early (max_iter=%i).'\n"
     ]
    }
   ],
   "source": [
    "mae_2nd, rmse_2nd = fold_scoring(df_list[1], meta, base_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lasso Regression</th>\n",
       "      <th>Multi-Layer Perceptron</th>\n",
       "      <th>Support Vector Regression</th>\n",
       "      <th>Support Vector Machine</th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>XG Boost</th>\n",
       "      <th>Stack Model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14.3973</td>\n",
       "      <td>15.2224</td>\n",
       "      <td>15.2739</td>\n",
       "      <td>18.9752</td>\n",
       "      <td>17.787</td>\n",
       "      <td>16.3779</td>\n",
       "      <td>14.7688</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Lasso Regression  Multi-Layer Perceptron  Support Vector Regression  \\\n",
       "0           14.3973                 15.2224                    15.2739   \n",
       "\n",
       "   Support Vector Machine  Random Forest  XG Boost  Stack Model  \n",
       "0                 18.9752         17.787   16.3779      14.7688  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mae_2nd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lasso Regression</th>\n",
       "      <th>Multi-Layer Perceptron</th>\n",
       "      <th>Support Vector Regression</th>\n",
       "      <th>Support Vector Machine</th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>XG Boost</th>\n",
       "      <th>Stack Model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>27.762</td>\n",
       "      <td>27.896</td>\n",
       "      <td>28.0028</td>\n",
       "      <td>36.2795</td>\n",
       "      <td>36.8735</td>\n",
       "      <td>40.5363</td>\n",
       "      <td>29.2265</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Lasso Regression  Multi-Layer Perceptron  Support Vector Regression  \\\n",
       "0            27.762                  27.896                    28.0028   \n",
       "\n",
       "   Support Vector Machine  Random Forest  XG Boost  Stack Model  \n",
       "0                 36.2795        36.8735   40.5363      29.2265  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rmse_2nd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Tan\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:985: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\Tan\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:255: ConvergenceWarning: Solver terminated early (max_iter=30000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  warnings.warn('Solver terminated early (max_iter=%i).'\n"
     ]
    }
   ],
   "source": [
    "mae_3rd, rmse_3rd = fold_scoring(df_list[2], meta, base_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lasso Regression</th>\n",
       "      <th>Multi-Layer Perceptron</th>\n",
       "      <th>Support Vector Regression</th>\n",
       "      <th>Support Vector Machine</th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>XG Boost</th>\n",
       "      <th>Stack Model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11.719</td>\n",
       "      <td>12.4461</td>\n",
       "      <td>12.3813</td>\n",
       "      <td>13.716</td>\n",
       "      <td>13.4982</td>\n",
       "      <td>11.6168</td>\n",
       "      <td>11.5525</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Lasso Regression  Multi-Layer Perceptron  Support Vector Regression  \\\n",
       "0            11.719                 12.4461                    12.3813   \n",
       "\n",
       "   Support Vector Machine  Random Forest  XG Boost  Stack Model  \n",
       "0                  13.716        13.4982   11.6168      11.5525  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mae_3rd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lasso Regression</th>\n",
       "      <th>Multi-Layer Perceptron</th>\n",
       "      <th>Support Vector Regression</th>\n",
       "      <th>Support Vector Machine</th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>XG Boost</th>\n",
       "      <th>Stack Model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>20.6169</td>\n",
       "      <td>20.8541</td>\n",
       "      <td>20.8621</td>\n",
       "      <td>23.0835</td>\n",
       "      <td>22.6239</td>\n",
       "      <td>20.8051</td>\n",
       "      <td>20.4243</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Lasso Regression  Multi-Layer Perceptron  Support Vector Regression  \\\n",
       "0           20.6169                 20.8541                    20.8621   \n",
       "\n",
       "   Support Vector Machine  Random Forest  XG Boost  Stack Model  \n",
       "0                 23.0835        22.6239   20.8051      20.4243  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rmse_3rd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Tan\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:985: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\Tan\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:255: ConvergenceWarning: Solver terminated early (max_iter=30000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  warnings.warn('Solver terminated early (max_iter=%i).'\n"
     ]
    }
   ],
   "source": [
    "mae_4th, rmse_4th = fold_scoring(df_list[3], meta, base_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lasso Regression</th>\n",
       "      <th>Multi-Layer Perceptron</th>\n",
       "      <th>Support Vector Regression</th>\n",
       "      <th>Support Vector Machine</th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>XG Boost</th>\n",
       "      <th>Stack Model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12.7882</td>\n",
       "      <td>13.6052</td>\n",
       "      <td>13.4848</td>\n",
       "      <td>15.2743</td>\n",
       "      <td>14.7573</td>\n",
       "      <td>12.8028</td>\n",
       "      <td>12.7065</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Lasso Regression  Multi-Layer Perceptron  Support Vector Regression  \\\n",
       "0           12.7882                 13.6052                    13.4848   \n",
       "\n",
       "   Support Vector Machine  Random Forest  XG Boost  Stack Model  \n",
       "0                 15.2743        14.7573   12.8028      12.7065  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mae_4th"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lasso Regression</th>\n",
       "      <th>Multi-Layer Perceptron</th>\n",
       "      <th>Support Vector Regression</th>\n",
       "      <th>Support Vector Machine</th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>XG Boost</th>\n",
       "      <th>Stack Model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>21.8869</td>\n",
       "      <td>22.3795</td>\n",
       "      <td>22.0211</td>\n",
       "      <td>25.0217</td>\n",
       "      <td>25.5284</td>\n",
       "      <td>22.7118</td>\n",
       "      <td>21.8517</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Lasso Regression  Multi-Layer Perceptron  Support Vector Regression  \\\n",
       "0           21.8869                 22.3795                    22.0211   \n",
       "\n",
       "   Support Vector Machine  Random Forest  XG Boost  Stack Model  \n",
       "0                 25.0217        25.5284   22.7118      21.8517  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rmse_4th"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Tan\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:985: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\Tan\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:255: ConvergenceWarning: Solver terminated early (max_iter=30000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  warnings.warn('Solver terminated early (max_iter=%i).'\n"
     ]
    }
   ],
   "source": [
    "mae_5th, rmse_5th = fold_scoring(df_list[4], meta, base_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lasso Regression</th>\n",
       "      <th>Multi-Layer Perceptron</th>\n",
       "      <th>Support Vector Regression</th>\n",
       "      <th>Support Vector Machine</th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>XG Boost</th>\n",
       "      <th>Stack Model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12.3165</td>\n",
       "      <td>12.802</td>\n",
       "      <td>13.3891</td>\n",
       "      <td>14.5751</td>\n",
       "      <td>13.3869</td>\n",
       "      <td>11.8918</td>\n",
       "      <td>12.118</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Lasso Regression  Multi-Layer Perceptron  Support Vector Regression  \\\n",
       "0           12.3165                  12.802                    13.3891   \n",
       "\n",
       "   Support Vector Machine  Random Forest  XG Boost  Stack Model  \n",
       "0                 14.5751        13.3869   11.8918       12.118  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mae_5th"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lasso Regression</th>\n",
       "      <th>Multi-Layer Perceptron</th>\n",
       "      <th>Support Vector Regression</th>\n",
       "      <th>Support Vector Machine</th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>XG Boost</th>\n",
       "      <th>Stack Model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23.1006</td>\n",
       "      <td>22.9184</td>\n",
       "      <td>25.6771</td>\n",
       "      <td>40.0732</td>\n",
       "      <td>23.107</td>\n",
       "      <td>22.5064</td>\n",
       "      <td>22.9992</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Lasso Regression  Multi-Layer Perceptron  Support Vector Regression  \\\n",
       "0           23.1006                 22.9184                    25.6771   \n",
       "\n",
       "   Support Vector Machine  Random Forest  XG Boost  Stack Model  \n",
       "0                 40.0732         23.107   22.5064      22.9992  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rmse_5th"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Comparative CV Scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Putting all the scores of the various folds together."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lasso Regression</th>\n",
       "      <th>Multi-Layer Perceptron</th>\n",
       "      <th>Support Vector Regression</th>\n",
       "      <th>Support Vector Machine</th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>XG Boost</th>\n",
       "      <th>Stack Model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1st fold</th>\n",
       "      <td>11.2777</td>\n",
       "      <td>12.5906</td>\n",
       "      <td>12.2504</td>\n",
       "      <td>13.4830</td>\n",
       "      <td>13.2545</td>\n",
       "      <td>11.4165</td>\n",
       "      <td>11.0952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2nd fold</th>\n",
       "      <td>14.3973</td>\n",
       "      <td>15.2224</td>\n",
       "      <td>15.2739</td>\n",
       "      <td>18.9752</td>\n",
       "      <td>17.7870</td>\n",
       "      <td>16.3779</td>\n",
       "      <td>14.7688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3rd fold</th>\n",
       "      <td>11.7190</td>\n",
       "      <td>12.4461</td>\n",
       "      <td>12.3813</td>\n",
       "      <td>13.7160</td>\n",
       "      <td>13.4982</td>\n",
       "      <td>11.6168</td>\n",
       "      <td>11.5525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4th fold</th>\n",
       "      <td>12.7882</td>\n",
       "      <td>13.6052</td>\n",
       "      <td>13.4848</td>\n",
       "      <td>15.2743</td>\n",
       "      <td>14.7573</td>\n",
       "      <td>12.8028</td>\n",
       "      <td>12.7065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5th fold</th>\n",
       "      <td>12.3165</td>\n",
       "      <td>12.8020</td>\n",
       "      <td>13.3891</td>\n",
       "      <td>14.5751</td>\n",
       "      <td>13.3869</td>\n",
       "      <td>11.8918</td>\n",
       "      <td>12.1180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>12.4997</td>\n",
       "      <td>13.3333</td>\n",
       "      <td>13.3559</td>\n",
       "      <td>15.2047</td>\n",
       "      <td>14.5368</td>\n",
       "      <td>12.8212</td>\n",
       "      <td>12.4482</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>median</th>\n",
       "      <td>12.3165</td>\n",
       "      <td>12.8020</td>\n",
       "      <td>13.3891</td>\n",
       "      <td>14.5751</td>\n",
       "      <td>13.4982</td>\n",
       "      <td>11.8918</td>\n",
       "      <td>12.1180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stdev</th>\n",
       "      <td>1.2063</td>\n",
       "      <td>1.1471</td>\n",
       "      <td>1.2112</td>\n",
       "      <td>2.2248</td>\n",
       "      <td>1.9143</td>\n",
       "      <td>2.0578</td>\n",
       "      <td>1.4312</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Lasso Regression  Multi-Layer Perceptron  Support Vector Regression  \\\n",
       "1st fold           11.2777                 12.5906                    12.2504   \n",
       "2nd fold           14.3973                 15.2224                    15.2739   \n",
       "3rd fold           11.7190                 12.4461                    12.3813   \n",
       "4th fold           12.7882                 13.6052                    13.4848   \n",
       "5th fold           12.3165                 12.8020                    13.3891   \n",
       "mean               12.4997                 13.3333                    13.3559   \n",
       "median             12.3165                 12.8020                    13.3891   \n",
       "stdev               1.2063                  1.1471                     1.2112   \n",
       "\n",
       "          Support Vector Machine  Random Forest  XG Boost  Stack Model  \n",
       "1st fold                 13.4830        13.2545   11.4165      11.0952  \n",
       "2nd fold                 18.9752        17.7870   16.3779      14.7688  \n",
       "3rd fold                 13.7160        13.4982   11.6168      11.5525  \n",
       "4th fold                 15.2743        14.7573   12.8028      12.7065  \n",
       "5th fold                 14.5751        13.3869   11.8918      12.1180  \n",
       "mean                     15.2047        14.5368   12.8212      12.4482  \n",
       "median                   14.5751        13.4982   11.8918      12.1180  \n",
       "stdev                     2.2248         1.9143    2.0578       1.4312  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stack_scores(mae_1st, mae_2nd, mae_3rd, mae_4th, mae_5th)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lasso Regression</th>\n",
       "      <th>Multi-Layer Perceptron</th>\n",
       "      <th>Support Vector Regression</th>\n",
       "      <th>Support Vector Machine</th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>XG Boost</th>\n",
       "      <th>Stack Model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1st fold</th>\n",
       "      <td>18.9254</td>\n",
       "      <td>19.7555</td>\n",
       "      <td>19.4904</td>\n",
       "      <td>20.9506</td>\n",
       "      <td>19.9403</td>\n",
       "      <td>18.7335</td>\n",
       "      <td>18.5377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2nd fold</th>\n",
       "      <td>27.7620</td>\n",
       "      <td>27.8960</td>\n",
       "      <td>28.0028</td>\n",
       "      <td>36.2795</td>\n",
       "      <td>36.8735</td>\n",
       "      <td>40.5363</td>\n",
       "      <td>29.2265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3rd fold</th>\n",
       "      <td>20.6169</td>\n",
       "      <td>20.8541</td>\n",
       "      <td>20.8621</td>\n",
       "      <td>23.0835</td>\n",
       "      <td>22.6239</td>\n",
       "      <td>20.8051</td>\n",
       "      <td>20.4243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4th fold</th>\n",
       "      <td>21.8869</td>\n",
       "      <td>22.3795</td>\n",
       "      <td>22.0211</td>\n",
       "      <td>25.0217</td>\n",
       "      <td>25.5284</td>\n",
       "      <td>22.7118</td>\n",
       "      <td>21.8517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5th fold</th>\n",
       "      <td>23.1006</td>\n",
       "      <td>22.9184</td>\n",
       "      <td>25.6771</td>\n",
       "      <td>40.0732</td>\n",
       "      <td>23.1070</td>\n",
       "      <td>22.5064</td>\n",
       "      <td>22.9992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>22.4584</td>\n",
       "      <td>22.7607</td>\n",
       "      <td>23.2107</td>\n",
       "      <td>29.0817</td>\n",
       "      <td>25.6146</td>\n",
       "      <td>25.0586</td>\n",
       "      <td>22.6079</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>median</th>\n",
       "      <td>21.8869</td>\n",
       "      <td>22.3795</td>\n",
       "      <td>22.0211</td>\n",
       "      <td>25.0217</td>\n",
       "      <td>23.1070</td>\n",
       "      <td>22.5064</td>\n",
       "      <td>21.8517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stdev</th>\n",
       "      <td>3.3444</td>\n",
       "      <td>3.1308</td>\n",
       "      <td>3.5291</td>\n",
       "      <td>8.5323</td>\n",
       "      <td>6.5992</td>\n",
       "      <td>8.7989</td>\n",
       "      <td>4.0579</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Lasso Regression  Multi-Layer Perceptron  Support Vector Regression  \\\n",
       "1st fold           18.9254                 19.7555                    19.4904   \n",
       "2nd fold           27.7620                 27.8960                    28.0028   \n",
       "3rd fold           20.6169                 20.8541                    20.8621   \n",
       "4th fold           21.8869                 22.3795                    22.0211   \n",
       "5th fold           23.1006                 22.9184                    25.6771   \n",
       "mean               22.4584                 22.7607                    23.2107   \n",
       "median             21.8869                 22.3795                    22.0211   \n",
       "stdev               3.3444                  3.1308                     3.5291   \n",
       "\n",
       "          Support Vector Machine  Random Forest  XG Boost  Stack Model  \n",
       "1st fold                 20.9506        19.9403   18.7335      18.5377  \n",
       "2nd fold                 36.2795        36.8735   40.5363      29.2265  \n",
       "3rd fold                 23.0835        22.6239   20.8051      20.4243  \n",
       "4th fold                 25.0217        25.5284   22.7118      21.8517  \n",
       "5th fold                 40.0732        23.1070   22.5064      22.9992  \n",
       "mean                     29.0817        25.6146   25.0586      22.6079  \n",
       "median                   25.0217        23.1070   22.5064      21.8517  \n",
       "stdev                     8.5323         6.5992    8.7989       4.0579  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stack_scores(rmse_1st, rmse_2nd, rmse_3rd, rmse_4th, rmse_5th)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Stack Model has the **lowest mean and median MAE scores**, with Lasso Regression coming in second.\n",
    "\n",
    "The Stack Model has the **second lowest mean RMSE score**, behind Lasso Regression, but the **lowest median RMSE score**."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}