Problem Set 4b- ML-Preethi.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Problem Set 4b- ML -Preethi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loading the packages\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "from scipy.stats import norm\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.preprocessing import OrdinalEncoder\n",
    "from scipy import stats\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.linear_model import Lasso\n",
    "import matplotlib as mpl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('C:/Users/Preethi Abraham/Desktop/Brandeis Studies/Machine Learning and Data Analytics/KaggleAmes.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop a bunch of useless series\n",
    "bigDrop = False\n",
    "if (bigDrop):\n",
    "    df.drop(df.columns[[0,2,6, 7, 8,9,10,11,12,13,14,21,22,23,24,25,27,28,29,\n",
    "                        32,33,35,36,39,40,42,45,47,48,49,50,52,53,55,57,60,63,\n",
    "                        64,65,68,69,70,71,72,73,74,75,76,78,79]],axis=1,inplace=True)\n",
    "if(not(bigDrop)):\n",
    "    df.drop(df.columns[[0]],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        MSSubClass  LotFrontage        LotArea  OverallQual  OverallCond  \\\n",
      "count  1460.000000  1201.000000    1460.000000  1460.000000  1460.000000   \n",
      "mean     56.897260    70.049958   10516.828082     6.099315     5.575342   \n",
      "std      42.300571    24.284752    9981.264932     1.382997     1.112799   \n",
      "min      20.000000    21.000000    1300.000000     1.000000     1.000000   \n",
      "25%      20.000000    59.000000    7553.500000     5.000000     5.000000   \n",
      "50%      50.000000    69.000000    9478.500000     6.000000     5.000000   \n",
      "75%      70.000000    80.000000   11601.500000     7.000000     6.000000   \n",
      "max     190.000000   313.000000  215245.000000    10.000000     9.000000   \n",
      "\n",
      "         YearBuilt  YearRemodAdd   MasVnrArea   BsmtFinSF1   BsmtFinSF2  ...  \\\n",
      "count  1460.000000   1460.000000  1452.000000  1460.000000  1460.000000  ...   \n",
      "mean   1971.267808   1984.865753   103.685262   443.639726    46.549315  ...   \n",
      "std      30.202904     20.645407   181.066207   456.098091   161.319273  ...   \n",
      "min    1872.000000   1950.000000     0.000000     0.000000     0.000000  ...   \n",
      "25%    1954.000000   1967.000000     0.000000     0.000000     0.000000  ...   \n",
      "50%    1973.000000   1994.000000     0.000000   383.500000     0.000000  ...   \n",
      "75%    2000.000000   2004.000000   166.000000   712.250000     0.000000  ...   \n",
      "max    2010.000000   2010.000000  1600.000000  5644.000000  1474.000000  ...   \n",
      "\n",
      "        WoodDeckSF  OpenPorchSF  EnclosedPorch    3SsnPorch  ScreenPorch  \\\n",
      "count  1460.000000  1460.000000    1460.000000  1460.000000  1460.000000   \n",
      "mean     94.244521    46.660274      21.954110     3.409589    15.060959   \n",
      "std     125.338794    66.256028      61.119149    29.317331    55.757415   \n",
      "min       0.000000     0.000000       0.000000     0.000000     0.000000   \n",
      "25%       0.000000     0.000000       0.000000     0.000000     0.000000   \n",
      "50%       0.000000    25.000000       0.000000     0.000000     0.000000   \n",
      "75%     168.000000    68.000000       0.000000     0.000000     0.000000   \n",
      "max     857.000000   547.000000     552.000000   508.000000   480.000000   \n",
      "\n",
      "          PoolArea       MiscVal       MoSold       YrSold      SalePrice  \n",
      "count  1460.000000   1460.000000  1460.000000  1460.000000    1460.000000  \n",
      "mean      2.758904     43.489041     6.321918  2007.815753  180921.195890  \n",
      "std      40.177307    496.123024     2.703626     1.328095   79442.502883  \n",
      "min       0.000000      0.000000     1.000000  2006.000000   34900.000000  \n",
      "25%       0.000000      0.000000     5.000000  2007.000000  129975.000000  \n",
      "50%       0.000000      0.000000     6.000000  2008.000000  163000.000000  \n",
      "75%       0.000000      0.000000     8.000000  2009.000000  214000.000000  \n",
      "max     738.000000  15500.000000    12.000000  2010.000000  755000.000000  \n",
      "\n",
      "[8 rows x 37 columns]\n",
      "(1460, 80)\n"
     ]
    }
   ],
   "source": [
    "print(df.describe())\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop series with similar stuff\n",
    "# Shift param down to 0.95\n",
    "# 1.00 = don't drop anything !!!\n",
    "sameThresh = 1.00\n",
    "dropSameList = []\n",
    "for i in range(df.shape[1]):\n",
    "    xsame = np.mean( df.iloc[0,i]==df.iloc[:,i])\n",
    "    if(xsame>sameThresh):\n",
    "        dropSameList.append(i) \n",
    "df.drop(df.columns[[dropSameList]],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check NA's in each column\n",
    "percent_missing = df.isnull().sum() * 100 / len(df)\n",
    "missing_value_df = pd.DataFrame({'column_name': df.columns,\n",
    "                                 'percent_missing': percent_missing})\n",
    "drop_col = missing_value_df[missing_value_df.percent_missing > 50]\n",
    "missing_list = drop_col.column_name.values\n",
    "df.drop(missing_list,axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        MSSubClass  LotFrontage        LotArea  OverallQual  OverallCond  \\\n",
      "count  1460.000000  1460.000000    1460.000000  1460.000000  1460.000000   \n",
      "mean     56.897260    70.049958   10516.828082     6.099315     5.575342   \n",
      "std      42.300571    22.024023    9981.264932     1.382997     1.112799   \n",
      "min      20.000000    21.000000    1300.000000     1.000000     1.000000   \n",
      "25%      20.000000    60.000000    7553.500000     5.000000     5.000000   \n",
      "50%      50.000000    70.049958    9478.500000     6.000000     5.000000   \n",
      "75%      70.000000    79.000000   11601.500000     7.000000     6.000000   \n",
      "max     190.000000   313.000000  215245.000000    10.000000     9.000000   \n",
      "\n",
      "         YearBuilt  YearRemodAdd   MasVnrArea   BsmtFinSF1   BsmtFinSF2  ...  \\\n",
      "count  1460.000000   1460.000000  1460.000000  1460.000000  1460.000000  ...   \n",
      "mean   1971.267808   1984.865753   103.685262   443.639726    46.549315  ...   \n",
      "std      30.202904     20.645407   180.569112   456.098091   161.319273  ...   \n",
      "min    1872.000000   1950.000000     0.000000     0.000000     0.000000  ...   \n",
      "25%    1954.000000   1967.000000     0.000000     0.000000     0.000000  ...   \n",
      "50%    1973.000000   1994.000000     0.000000   383.500000     0.000000  ...   \n",
      "75%    2000.000000   2004.000000   164.250000   712.250000     0.000000  ...   \n",
      "max    2010.000000   2010.000000  1600.000000  5644.000000  1474.000000  ...   \n",
      "\n",
      "       SaleType_ConLw  SaleType_New  SaleType_Oth  SaleType_WD  \\\n",
      "count     1460.000000   1460.000000   1460.000000  1460.000000   \n",
      "mean         0.003425      0.083562      0.002055     0.867808   \n",
      "std          0.058440      0.276824      0.045299     0.338815   \n",
      "min          0.000000      0.000000      0.000000     0.000000   \n",
      "25%          0.000000      0.000000      0.000000     1.000000   \n",
      "50%          0.000000      0.000000      0.000000     1.000000   \n",
      "75%          0.000000      0.000000      0.000000     1.000000   \n",
      "max          1.000000      1.000000      1.000000     1.000000   \n",
      "\n",
      "       SaleCondition_Abnorml  SaleCondition_AdjLand  SaleCondition_Alloca  \\\n",
      "count            1460.000000            1460.000000           1460.000000   \n",
      "mean                0.069178               0.002740              0.008219   \n",
      "std                 0.253844               0.052289              0.090317   \n",
      "min                 0.000000               0.000000              0.000000   \n",
      "25%                 0.000000               0.000000              0.000000   \n",
      "50%                 0.000000               0.000000              0.000000   \n",
      "75%                 0.000000               0.000000              0.000000   \n",
      "max                 1.000000               1.000000              1.000000   \n",
      "\n",
      "       SaleCondition_Family  SaleCondition_Normal  SaleCondition_Partial  \n",
      "count           1460.000000           1460.000000            1460.000000  \n",
      "mean               0.013699              0.820548               0.085616  \n",
      "std                0.116277              0.383862               0.279893  \n",
      "min                0.000000              0.000000               0.000000  \n",
      "25%                0.000000              1.000000               0.000000  \n",
      "50%                0.000000              1.000000               0.000000  \n",
      "75%                0.000000              1.000000               0.000000  \n",
      "max                1.000000              1.000000               1.000000  \n",
      "\n",
      "[8 rows x 276 columns]\n"
     ]
    }
   ],
   "source": [
    "# generate dummies\n",
    "df = pd.get_dummies(df)\n",
    "# fill na's with means\n",
    "df = df.fillna(df.mean())\n",
    "print(df.describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#change y to SalesPrice log \n",
    "# y = np.log(df.SalePrice)\n",
    "y = df.SalePrice\n",
    "\n",
    "#drop original dependent variable\n",
    "X = df.drop(['SalePrice'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>OverallQual</th>\n",
       "      <th>OverallCond</th>\n",
       "      <th>YearBuilt</th>\n",
       "      <th>YearRemodAdd</th>\n",
       "      <th>MasVnrArea</th>\n",
       "      <th>BsmtFinSF1</th>\n",
       "      <th>BsmtFinSF2</th>\n",
       "      <th>...</th>\n",
       "      <th>SaleType_ConLw</th>\n",
       "      <th>SaleType_New</th>\n",
       "      <th>SaleType_Oth</th>\n",
       "      <th>SaleType_WD</th>\n",
       "      <th>SaleCondition_Abnorml</th>\n",
       "      <th>SaleCondition_AdjLand</th>\n",
       "      <th>SaleCondition_Alloca</th>\n",
       "      <th>SaleCondition_Family</th>\n",
       "      <th>SaleCondition_Normal</th>\n",
       "      <th>SaleCondition_Partial</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>60</td>\n",
       "      <td>65.0</td>\n",
       "      <td>8450</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>2003</td>\n",
       "      <td>2003</td>\n",
       "      <td>196.0</td>\n",
       "      <td>706</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20</td>\n",
       "      <td>80.0</td>\n",
       "      <td>9600</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>1976</td>\n",
       "      <td>1976</td>\n",
       "      <td>0.0</td>\n",
       "      <td>978</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>60</td>\n",
       "      <td>68.0</td>\n",
       "      <td>11250</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>2001</td>\n",
       "      <td>2002</td>\n",
       "      <td>162.0</td>\n",
       "      <td>486</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>70</td>\n",
       "      <td>60.0</td>\n",
       "      <td>9550</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>1915</td>\n",
       "      <td>1970</td>\n",
       "      <td>0.0</td>\n",
       "      <td>216</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>60</td>\n",
       "      <td>84.0</td>\n",
       "      <td>14260</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>2000</td>\n",
       "      <td>2000</td>\n",
       "      <td>350.0</td>\n",
       "      <td>655</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 276 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   MSSubClass  LotFrontage  LotArea  OverallQual  OverallCond  YearBuilt  \\\n",
       "0          60         65.0     8450            7            5       2003   \n",
       "1          20         80.0     9600            6            8       1976   \n",
       "2          60         68.0    11250            7            5       2001   \n",
       "3          70         60.0     9550            7            5       1915   \n",
       "4          60         84.0    14260            8            5       2000   \n",
       "\n",
       "   YearRemodAdd  MasVnrArea  BsmtFinSF1  BsmtFinSF2  ...  SaleType_ConLw  \\\n",
       "0          2003       196.0         706           0  ...               0   \n",
       "1          1976         0.0         978           0  ...               0   \n",
       "2          2002       162.0         486           0  ...               0   \n",
       "3          1970         0.0         216           0  ...               0   \n",
       "4          2000       350.0         655           0  ...               0   \n",
       "\n",
       "   SaleType_New  SaleType_Oth  SaleType_WD  SaleCondition_Abnorml  \\\n",
       "0             0             0            1                      0   \n",
       "1             0             0            1                      0   \n",
       "2             0             0            1                      0   \n",
       "3             0             0            1                      1   \n",
       "4             0             0            1                      0   \n",
       "\n",
       "   SaleCondition_AdjLand  SaleCondition_Alloca  SaleCondition_Family  \\\n",
       "0                      0                     0                     0   \n",
       "1                      0                     0                     0   \n",
       "2                      0                     0                     0   \n",
       "3                      0                     0                     0   \n",
       "4                      0                     0                     0   \n",
       "\n",
       "   SaleCondition_Normal  SaleCondition_Partial  \n",
       "0                     1                      0  \n",
       "1                     1                      0  \n",
       "2                     1                      0  \n",
       "3                     0                      0  \n",
       "4                     1                      0  \n",
       "\n",
       "[5 rows x 276 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this very simple problem, I would just like you to extend your analysis of the Ames Housing data to include a random forest regressor.\n",
    "\n",
    "Set up the random forest regressor and estimate optimal parameters using a limited grid search (limited means of restricted the search, so it should finish on your machine in reasonable time).\n",
    "\n",
    "Use a parameter grid of the following,\n",
    "\n",
    "param_grid={'n_estimators':[100],'max_depth':[15, 50, 100],'max_features':[50,100,200]}\n",
    "\n",
    "Find the best model based on test scores (R-squared) with 100 randomized cross validations, and set the test size to 0.2 of the sample.\n",
    "\n",
    "Perform one monte-carlo randomized cross validation (nmc=100, test size = 0.2) on the full sample and report the mean train and test scores.  How does this model compare with the others you used in pset4?\n",
    "\n",
    "Note:  I did not ask about data rescaling here?  Why not?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import ShuffleSplit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=ShuffleSplit(n_splits=100, random_state=None, test_size=0.2, train_size=None),\n",
       "             error_score=nan,\n",
       "             estimator=RandomForestRegressor(bootstrap=True, ccp_alpha=0.0,\n",
       "                                             criterion='mse', max_depth=None,\n",
       "                                             max_features='auto',\n",
       "                                             max_leaf_nodes=None,\n",
       "                                             max_samples=None,\n",
       "                                             min_impurity_decrease=0.0,\n",
       "                                             min_impurity_split=None,\n",
       "                                             min_samples_leaf=1,\n",
       "                                             min_samples_split=2,\n",
       "                                             min_weight_fraction_leaf=0.0,\n",
       "                                             n_estimators=100, n_jobs=None,\n",
       "                                             oob_score=False, random_state=None,\n",
       "                                             verbose=0, warm_start=False),\n",
       "             iid='deprecated', n_jobs=None,\n",
       "             param_grid={'max_depth': [15, 50, 100],\n",
       "                         'max_features': [50, 100, 200],\n",
       "                         'n_estimators': [100]},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "             scoring='r2', verbose=0)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score_used = 'r2' \n",
    "param_grid={'n_estimators':[100],'max_depth':[15, 50, 100],'max_features':[50,100,200]}\n",
    "shuffle_split = ShuffleSplit(test_size=0.2, n_splits=100)\n",
    "grid_search=GridSearchCV(RandomForestRegressor(),param_grid,cv=shuffle_split,scoring=score_used, \n",
    "                              return_train_score=True)\n",
    "grid_search.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  param_max_depth param_max_features param_n_estimators  mean_test_score\n",
      "0              15                 50                100         0.862981\n",
      "1              15                100                100         0.863248\n",
      "2              15                200                100         0.858860\n",
      "3              50                 50                100         0.862197\n",
      "4              50                100                100         0.863186\n",
      "5              50                200                100         0.859305\n",
      "6             100                 50                100         0.861691\n",
      "7             100                100                100         0.864038\n",
      "8             100                200                100         0.859766\n"
     ]
    }
   ],
   "source": [
    "results = pd.DataFrame(grid_search.cv_results_)\n",
    "print(results[['param_max_depth','param_max_features','param_n_estimators','mean_test_score']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def MCtraintest(nmc,X,y,modelObj,testFrac):\n",
    "    # make space to store scores\n",
    "    trainScore = np.zeros(nmc)\n",
    "    testScore  = np.zeros(nmc)\n",
    "    for i in range(nmc):\n",
    "        X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=testFrac)\n",
    "        modelObj.fit(X_train,y_train)\n",
    "        trainScore[i] = modelObj.score(X_train,y_train)\n",
    "        testScore[i]  = modelObj.score(X_test,y_test)\n",
    "    return trainScore,testScore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Score= 0.9796217460635174\n",
      "Test Score= 0.8552179157544632\n"
     ]
    }
   ],
   "source": [
    "rf=RandomForestRegressor(n_estimators=100,max_depth=15)\n",
    "trainS, testS = MCtraintest(100,X,y,rf,0.2)\n",
    "print(\"Train Score=\",np.mean(trainS))\n",
    "print(\"Test Score=\",np.mean(testS))\n",
    "\n",
    "#The training and testing scores are really high."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In Problem Set 4 the results were as follows:\n",
    "\n",
    "1. Linear Regression:\n",
    "Full sample R-squared:\n",
    "0.9313624499887028\n",
    "\n",
    "2. Ridge Regression:\n",
    "\n",
    "           rank_test_score     mean_test_score  param_alpha\n",
    "                   3         0.771688         0.5\n",
    "                   2         0.775871           1\n",
    "                   1         0.779921           2\n",
    "\n",
    "3. Ridge Regression with StandardScaler()\n",
    "\n",
    "             rank_test_score  mean_test_score    param_ridge__alpha\n",
    "                   3         0.781877                0.5\n",
    "                    2         0.782923                  1\n",
    "                    1         0.784711                  2\n",
    "\n",
    "\n",
    "The results obtained from RandomForestRegressor() are much higher compared to the results from pset4. The test scores are in the range of 0.859 to 0.864.\n",
    "\n",
    "Since RandomForestRegression is a tree based algorithm, we do not need to scale the data. This algorithm is not sensitive to magnitude of the input variables. Hence, scaling of features are not required. We only need to scale data when we use algorithms like KNN,Support Vector Machine,  Lasso and Ridge Regressions since these algorithms are sensitive to the input variables."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}