supervised_learning/decision-tree/demo/dt_regression_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_leaf(data):\n",
    "    '''Returns the most frequent label occuring in the\n",
    "        dependent variable data'''\n",
    "    y = data[:,-1]\n",
    "    label = np.mean(y)\n",
    "    return label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_feature_types(df):\n",
    "    n_unique_value_threshold = 5\n",
    "    feature_types = []\n",
    "    for index in range(df.shape[1]):\n",
    "        datatype = str(df.iloc[:,index].dtype)\n",
    "        cond1_for_num = 'float' in datatype or 'int' in datatype\n",
    "        cond2_for_num = len(np.unique(df.iloc[:,index].values)) > n_unique_value_threshold\n",
    "        if cond1_for_num and cond2_for_num:\n",
    "            feature_type = 'numerical'\n",
    "        else:\n",
    "            feature_type = 'categorical'\n",
    "        feature_types.append(feature_type)\n",
    "    return feature_types\n",
    "\n",
    "\n",
    "def get_potential_splits(data,feature_types):\n",
    "    '''Return a dictionary of potential splits with keys \n",
    "        corresponding to columns of feature matrix and \n",
    "        values corresponding to pontial spilts in each \n",
    "        column''' \n",
    "    X = data[:,:-1] #Extracting the feature matrix\n",
    "    potential_splits = {}\n",
    "    for column_index in range(X.shape[1]):\n",
    "        unique_values = np.unique(X[:,column_index])\n",
    "        if feature_types[column_index] == 'numerical' and len(unique_values) > 1:\n",
    "            potential_splits[column_index] = []\n",
    "            for index in range(1,len(unique_values)): #Skiping the first index\n",
    "                current_value = unique_values[index]\n",
    "                previous_value = unique_values[index - 1]\n",
    "                potential_split = (current_value + previous_value) / 2  \n",
    "                potential_splits[column_index].append(potential_split)\n",
    "        elif feature_types[column_index] == 'categorical' and len(unique_values) > 1:\n",
    "            potential_splits[column_index] = list(unique_values)\n",
    "    return potential_splits\n",
    "\n",
    "def split_data(data,split_col,split_value,feature_types):\n",
    "    feature_type = feature_types[split_col]\n",
    "    \n",
    "    if feature_type == 'numerical':\n",
    "        data_left = data[data[:,split_col] <= split_value]\n",
    "        data_right = data[data[:,split_col] > split_value ]\n",
    "\n",
    "    elif feature_type == 'categorical':        \n",
    "        data_left = data[data[:,split_col] == split_value]\n",
    "        data_right = data[data[:,split_col] != split_value]\n",
    "    \n",
    "    return data_left, data_right\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def leaf_mse(y):\n",
    "    if len(y) == 0:\n",
    "        mse = 0\n",
    "    else:\n",
    "        leaf_mean = np.mean(y)\n",
    "        mse = np.mean((leaf_mean - y)**2)\n",
    "    return mse\n",
    "\n",
    "def split_mse(data_left,data_right):\n",
    "    n = len(data_left) + len(data_right)\n",
    "    p_left = len(data_left)/n\n",
    "    p_right = len(data_right)/n\n",
    "    split_mse = p_left*leaf_mse(data_left[:,-1]) + p_right*leaf_mse(data_right[:,-1])\n",
    "    return split_mse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "def determine_best_split(data,potential_splits,feature_types):\n",
    "    first_iter = True #making sure that this if statement is executed atleast once, otherwise an arbitrarily high values must be set\n",
    "    for column_index in potential_splits:\n",
    "        for value in potential_splits[column_index]:\n",
    "            data_left,data_right = split_data(data,column_index,value,feature_types)\n",
    "            current_mse = split_mse(data_left,data_right)\n",
    "            if first_iter or current_mse <= mse_best_split:\n",
    "                first_iter = False \n",
    "                mse_best_split = current_mse\n",
    "                best_split_col = column_index\n",
    "                best_split_value = value\n",
    "    return best_split_col,best_split_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decision_tree_regression(data,min_samples,max_depth,column_headers,feature_types,counter):\n",
    "    potential_splits = get_potential_splits(data,feature_types)\n",
    "    #Base case, if the data is pure,\n",
    "              # if there are no potential splits\n",
    "              # if max depth is reached, classify the data\n",
    "    if (len(data) < min_samples) or (counter == max_depth) or len(potential_splits) == 0:\n",
    "        return create_leaf(data)                                                         # if there are no potential splits,\n",
    "                                                                                      # classify the data\n",
    "    #Recrusive Part, if base case is not satisfied\n",
    "    else:\n",
    "        counter += 1\n",
    "        \n",
    "        #helper functions\n",
    "        split_col,split_val = determine_best_split(data,potential_splits,feature_types)\n",
    "        data_left, data_right = split_data(data,split_col,split_val,feature_types) #f\n",
    "        \n",
    "        col_name = column_headers[split_col]\n",
    "        feat_type = feature_types[split_col]\n",
    "        \n",
    "        if feat_type == 'numerical':\n",
    "            question = f'{col_name} <= {split_val}'\n",
    "        else: #feat_type is categorical\n",
    "            question = f'{col_name} = {split_val}'\n",
    "            \n",
    "        subtree = {question: []}\n",
    "        yes_ans = decision_tree_regression(data_left,min_samples,max_depth,column_headers,feature_types,counter)\n",
    "        no_ans = decision_tree_regression(data_right,min_samples,max_depth,column_headers,feature_types,counter)\n",
    "        \n",
    "        #if after splitting, both dataset labels are same then instead of the subtree;\n",
    "        #being a dict and creating further nodes, return any one of the answer and create a leaf\n",
    "        #This has high revelence in a classification problem, not so much in regression\n",
    "        if yes_ans == no_ans: \n",
    "            subtree = yes_ans\n",
    "        else:\n",
    "            subtree[question].append(yes_ans)\n",
    "            subtree[question].append(no_ans)\n",
    "\n",
    "    return subtree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def single_predict(example,tree):\n",
    "    question = list(tree.keys())[0]\n",
    "    col_name,operator,value = question.split()\n",
    "    if operator == '<=':\n",
    "        if str(example[col_name]) <= value:\n",
    "            answer = tree[question][0]\n",
    "        else:\n",
    "            answer = tree[question][1]\n",
    "    else:\n",
    "         if str(example[col_name]) == value:\n",
    "            answer = tree[question][0]\n",
    "         else:\n",
    "            answer = tree[question][1]\n",
    "    if not isinstance(answer, dict):\n",
    "        return answer\n",
    "    else:\n",
    "        residual_tree = answer\n",
    "        return single_predict(example,residual_tree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DecisionTreeRegressor():\n",
    "    def __init__(self,max_depth=None,min_samples=3):\n",
    "        self.tree = None\n",
    "        self.max_depth = max_depth\n",
    "        self.min_samples = min_samples\n",
    "        self.column_headers = None\n",
    "    \n",
    "    def train(self,X,y):\n",
    "        '''The feature matrix(X) must be a data frame, so that the column\n",
    "        headers may be extracted'''\n",
    "        column_headers = X.columns\n",
    "        feature_types = get_feature_types(X)\n",
    "        self.counter = 0\n",
    "        X = X.values\n",
    "        y = y.values.reshape(y.size,-1)\n",
    "        data = np.hstack((X,y)) #Converting the dataframes into an array\n",
    "        \n",
    "        self.tree = decision_tree_regression(data,self.min_samples,self.max_depth,column_headers,feature_types,self.counter)\n",
    "        return self.tree\n",
    "    \n",
    "    def predict(self,X):\n",
    "        predictions = []\n",
    "        for index in range(len(X)):\n",
    "            example = X.iloc[index]\n",
    "            prediction = single_predict(example,self.tree)\n",
    "            predictions.append(prediction)\n",
    "        predictions = np.array(predictions)\n",
    "        return predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('datasets/house_prices.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.dropna(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = data.iloc[:,:-1]\n",
    "y = data.iloc[:,-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = DecisionTreeRegressor(max_depth=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'OverallQual <= 7.5': [{'OverallQual <= 6.5': [140383.97587719298,\n",
       "    207716.42319749217]},\n",
       "  {'OverallQual <= 8.5': [274735.53571428574, 388486.08196721313]}]}"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.train(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}