feature_importances.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "18c16443-2b4f-4a08-abb4-09299e63b5c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Libraries imported\n"
     ]
    }
   ],
   "source": [
    "# Evaluate importances of different features\n",
    "\n",
    "# In the metrics evaluation, random forest is chosen as \n",
    "# the model to be used. \n",
    "# So, the importances of features will be done on the\n",
    "# random forest only.\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from configparser import ConfigParser\n",
    "\n",
    "#import warnings\n",
    "print(\"Libraries imported\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "326998d1-d966-41cd-98c3-da1639762d3b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['polynomial', 'sgd', 'svr', 'forest', 'knn']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Open the importances file,\n",
    "\n",
    "config = ConfigParser(allow_no_value=True)\n",
    "filename = \"output_all/importance.conf\"\n",
    "config.read(filename)\n",
    "config.sections()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f82419fd-8f93-4375-857a-870d1b5c02e0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['features', 'label']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Also read in the name of features used\n",
    "\n",
    "config_feature = ConfigParser(allow_no_value=True)\n",
    "filename = \"output_all/features.conf\"\n",
    "config_feature.read(filename)\n",
    "config_feature.sections()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "78770ec1-2530-403e-8639-691e56b15bdf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "direct_admission = True\n",
      "cca = True\n",
      "learning_style = True\n",
      "tuition = True\n",
      "hours_per_week = True\n",
      "attendance_rate = True\n",
      "sleep_duration = True\n",
      "sleep_enough = True\n",
      "attendance_enough = True\n",
      "number_of_siblings_0 = True\n",
      "number_of_siblings_1 = True\n",
      "number_of_siblings_2 = True\n",
      "['direct_admission', 'cca', 'learning_style', 'tuition', 'hours_per_week', 'attendance_rate', 'sleep_duration', 'sleep_enough', 'attendance_enough', 'number_of_siblings_0', 'number_of_siblings_1', 'number_of_siblings_2']\n"
     ]
    }
   ],
   "source": [
    "# Get the feature_names selected (True)\n",
    "\n",
    "feature_names = []\n",
    "\n",
    "for k, v in config_feature['features'].items():\n",
    "    # append only the selected features\n",
    "    if v == 'True':\n",
    "        feature_names.append(k)\n",
    "    print(f\"{k} = {v}\")\n",
    "\n",
    "print(feature_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2a385e92-6307-477f-81cc-e3b6293e6aa2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[2.82414419 1.62504549 2.46429073 2.04280771 4.12932336 2.51049243\\n0.19465569 0.09617319 0.48906955 2.6646175  1.18457411 0.44382001]'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ConfigParser reads this in as a string \n",
    "s = config['forest']['importances_mean']\n",
    "s\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b273b851-9c67-446c-b001-8b60c75035c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.82414419 1.62504549 2.46429073 2.04280771 4.12932336 2.51049243\\n0.19465569 0.09617319 0.48906955 2.6646175  1.18457411 0.44382001'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s = s.strip(\"[]\")\n",
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3e6cc43e-10f5-481c-bffa-2edfb6b29dca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.82414419 1.62504549 2.46429073 2.04280771 4.12932336 2.51049243 0.19465569 0.09617319 0.48906955 2.6646175  1.18457411 0.44382001'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s = s.replace(\"\\n\", \" \")\n",
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "78ef2400-61dc-4cd7-91ad-7fd081065445",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['2.82414419', '1.62504549', '2.46429073', '2.04280771', '4.12932336', '2.51049243', '0.19465569', '0.09617319', '0.48906955', '2.6646175', '1.18457411', '0.44382001']\n",
      "[2.82414419, 1.62504549, 2.46429073, 2.04280771, 4.12932336, 2.51049243, 0.19465569, 0.09617319, 0.48906955, 2.6646175, 1.18457411, 0.44382001]\n"
     ]
    }
   ],
   "source": [
    "# Get the mean of importances\n",
    "impt = []\n",
    "\n",
    "value_list = s.split()\n",
    "print(value_list)\n",
    "for i in value_list:\n",
    "    impt.append(float(i))\n",
    "\n",
    "print(impt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "221082ce-d5d6-4085-8e04-57d721a0ae29",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make into dataframe for plotting\n",
    "impt_df = pd.DataFrame()\n",
    "impt_df['feature_names'] = feature_names\n",
    "impt_df['importances_mean'] = impt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "feea8cdc-f821-4023-9eaa-370d0c04958c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature_names</th>\n",
       "      <th>importances_mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>hours_per_week</td>\n",
       "      <td>4.129323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>direct_admission</td>\n",
       "      <td>2.824144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>number_of_siblings_0</td>\n",
       "      <td>2.664617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>attendance_rate</td>\n",
       "      <td>2.510492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>learning_style</td>\n",
       "      <td>2.464291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>tuition</td>\n",
       "      <td>2.042808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cca</td>\n",
       "      <td>1.625045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>number_of_siblings_1</td>\n",
       "      <td>1.184574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>attendance_enough</td>\n",
       "      <td>0.489070</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>number_of_siblings_2</td>\n",
       "      <td>0.443820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>sleep_duration</td>\n",
       "      <td>0.194656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>sleep_enough</td>\n",
       "      <td>0.096173</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           feature_names  importances_mean\n",
       "0         hours_per_week          4.129323\n",
       "1       direct_admission          2.824144\n",
       "2   number_of_siblings_0          2.664617\n",
       "3        attendance_rate          2.510492\n",
       "4         learning_style          2.464291\n",
       "5                tuition          2.042808\n",
       "6                    cca          1.625045\n",
       "7   number_of_siblings_1          1.184574\n",
       "8      attendance_enough          0.489070\n",
       "9   number_of_siblings_2          0.443820\n",
       "10        sleep_duration          0.194656\n",
       "11          sleep_enough          0.096173"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Rank the importances\n",
    "\n",
    "impt_df.sort_values(by='importances_mean', ascending=False, ignore_index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d756ded-fb5a-4c72-9db4-701c98643dee",
   "metadata": {},
   "source": [
    "From the ranking, the revision `hours_per_week` is the highest. \n",
    "\n",
    "`direct_admission` ranks second in the importances. The boxplot in EDA also show that its mean is different from those that is not `direct_admission`.\n",
    "\n",
    "The number_of_siblings had been encoded by `get_dummies`, hence the `_0`, `_1` and `_2`. 0 number of siblings has the highest mean `final_test` score. This is also reflected in its importances to make regression.\n",
    "\n",
    "The `number_of_siblings_0` and `direct_admission` rank the second and the third place respectively. It is surprising that `number_of_siblings_0` can be so important to the random forest.\n",
    "\n",
    "For `sleep_duration` and `sleep_enough`, the interquartile range captures most of the samples, so this feature is general and do not contribute much to score regression. Thus, their rankings are rather low.\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}