Feature Engineering Part2-Handling missing Values.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5055bcbd",
   "metadata": {},
   "source": [
    "Mean Median Mode Imputation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4b5ecae0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d0a4ceb8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv('titanic.csv',usecols=['Age','Fare','Survived'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "933de394",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Age</th>\n",
       "      <th>Fare</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>7.2500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>38.0</td>\n",
       "      <td>71.2833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>26.0</td>\n",
       "      <td>7.9250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>53.1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>8.0500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Survived   Age     Fare\n",
       "0         0  22.0   7.2500\n",
       "1         1  38.0  71.2833\n",
       "2         1  26.0   7.9250\n",
       "3         1  35.0  53.1000\n",
       "4         0  35.0   8.0500"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9d76a79e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Survived    0.000000\n",
       "Age         0.198653\n",
       "Fare        0.000000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "###% of missing values\n",
    "\n",
    "df.isnull().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "89a08fb4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def impute_nan(df,variable,median):\n",
    "    df[variable+\"_median\"]=df[variable].fillna(median)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "00c36a6d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28.0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "median=df.Age.median()\n",
    "median"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "03c644fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "impute_nan(df,'Age',median)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "17fabcdc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Age</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Age_median</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>38.0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>38.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>26.0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>26.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>35.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>35.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Survived   Age     Fare  Age_median\n",
       "0         0  22.0   7.2500        22.0\n",
       "1         1  38.0  71.2833        38.0\n",
       "2         1  26.0   7.9250        26.0\n",
       "3         1  35.0  53.1000        35.0\n",
       "4         0  35.0   8.0500        35.0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "4a4c6fcf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Age</th>\n",
       "      <th>Fare</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>7.2500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>38.0</td>\n",
       "      <td>71.2833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>26.0</td>\n",
       "      <td>7.9250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>53.1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>8.0500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Survived   Age     Fare\n",
       "0         0  22.0   7.2500\n",
       "1         1  38.0  71.2833\n",
       "2         1  26.0   7.9250\n",
       "3         1  35.0  53.1000\n",
       "4         0  35.0   8.0500"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv('titanic.csv',usecols=['Age','Fare','Survived'])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "943bbf78",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Survived      0\n",
       "Age         177\n",
       "Fare          0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "9d27e288",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "423    28.00\n",
       "177    50.00\n",
       "305     0.92\n",
       "292    36.00\n",
       "889    26.00\n",
       "       ...  \n",
       "539    22.00\n",
       "267    25.00\n",
       "352    15.00\n",
       "99     34.00\n",
       "689    15.00\n",
       "Name: Age, Length: 177, dtype: float64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Age'].dropna().sample(df['Age'].isnull().sum(),random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c1a791f4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "129    45.0\n",
       "Name: Age, dtype: float64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#get a sample from a column\n",
    "\n",
    "#1Step1 drop missing values\n",
    "df['Age'].dropna().sample()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24d253ff",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "5abc935a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def impute_nan(df,variable,median):\n",
    "    df[variable+\"_Random_sampling\"]=df[variable].fillna(median)\n",
    "    df[variable+ \"_random\"]=df[variable]\n",
    "    random_sample=df[variable].dropna().sample(df[variable].isnull().sum(),random_state=0)    \n",
    "    ##pandas needs to have same index to merge\n",
    "    \n",
    "    random_sample_index=df[df[variable].isnull()].index\n",
    "    \n",
    "    df.loc[df[variable].isnull(),variable+ '_random']=random_sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "58e33bb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "impute_nan(df,'Age',median)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "c4c72f8e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Age</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Age_Random_sampling</th>\n",
       "      <th>Age_random</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>22.0</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>38.0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>38.0</td>\n",
       "      <td>38.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>26.0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>26.0</td>\n",
       "      <td>26.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>35.0</td>\n",
       "      <td>35.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>35.0</td>\n",
       "      <td>35.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Survived   Age     Fare  Age_Random_sampling  Age_random\n",
       "0         0  22.0   7.2500                 22.0        22.0\n",
       "1         1  38.0  71.2833                 38.0        38.0\n",
       "2         1  26.0   7.9250                 26.0        26.0\n",
       "3         1  35.0  53.1000                 35.0        35.0\n",
       "4         0  35.0   8.0500                 35.0        35.0"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a1e91f3",
   "metadata": {},
   "source": [
    "Capturing NAN with a new feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "dee327e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Age</th>\n",
       "      <th>Fare</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>7.2500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>38.0</td>\n",
       "      <td>71.2833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>26.0</td>\n",
       "      <td>7.9250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>53.1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>8.0500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Survived   Age     Fare\n",
       "0         0  22.0   7.2500\n",
       "1         1  38.0  71.2833\n",
       "2         1  26.0   7.9250\n",
       "3         1  35.0  53.1000\n",
       "4         0  35.0   8.0500"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv('titanic.csv',usecols=['Age','Fare','Survived'])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "ccfdedd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Age_nan'] = np.where(df['Age'].isnull(),1,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "e30e062c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "1f26fa11",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Age</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Age_nan</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>38.0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>26.0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Survived   Age     Fare  Age_nan\n",
       "0         0  22.0   7.2500        0\n",
       "1         1  38.0  71.2833        0\n",
       "2         1  26.0   7.9250        0\n",
       "3         1  35.0  53.1000        0\n",
       "4         0  35.0   8.0500        0"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83a62a12",
   "metadata": {},
   "source": [
    "End of Distribution Imputation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "cfe0c269",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Age</th>\n",
       "      <th>Fare</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>7.2500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>38.0</td>\n",
       "      <td>71.2833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>26.0</td>\n",
       "      <td>7.9250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>53.1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>8.0500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Survived   Age     Fare\n",
       "0         0  22.0   7.2500\n",
       "1         1  38.0  71.2833\n",
       "2         1  26.0   7.9250\n",
       "3         1  35.0  53.1000\n",
       "4         0  35.0   8.0500"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv('titanic.csv',usecols=['Age','Fare','Survived'])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "7507c504",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAARRklEQVR4nO3dXYxcd33G8e8PQ0vwgl+asHIdVAfVSkG4SfAqhKaqdmNA5kVxLpoqCJAtpfINtKFy1Tqt1IqLqrloUHuBqlpAbZU220BJYxkpwTLZVlQVYEPATp3UlFghTmpDahs2jShOf73Ys/F0PbszZ152zt/+fqTVzDlz5syzM+PHZ/9zzpzITCRJ5XnVqANIknpjgUtSoSxwSSqUBS5JhbLAJalQr17OB7v66qtzw4YNte/34osvsnLlysEH6pO56mlqLmhuNnPV09Rc0F+2I0eO/DAzr7nkhsxctp/NmzdnLx577LGe7jds5qqnqbkym5vNXPU0NVdmf9mAw9mmUx1CkaRCWeCSVCgLXJIKZYFLUqEscEkqlAUuSYWywCWpUBa4JBXKApekQi3rofQajA27v9R2/sn73r/MSSSNUlcFHhEngR8DLwMXMnMiItYCfw9sAE4Cv5GZZ4cTU5K0UJ0hlKnMvDEzJ6rp3cChzNwIHKqmJUnLpJ8x8G3Avur6PuCOvtNIkroW2cVJjSPiaeAskMBfZeaeiDiXmatbljmbmWva3HcnsBNgfHx88/T0dO2Qs7OzjI2N1b7fsI0q19FT59vO37R+FeDz1YumZjNXPU3NBf1lm5qaOtIy+vGKbj/EvDUzn4uINwIHI+LJbh84M/cAewAmJiZycnKy27u+YmZmhl7uN2yjyrVjsQ8xPzQJ+Hz1oqnZzFVPU3PBcLJ1NYSSmc9Vl2eAh4CbgdMRsQ6gujwz0GSSpCV1LPCIWBkRr5+/DrwHOAbsB7ZXi20HHh5WSEnSpboZQhkHHoqI+eX/LjMfiYhvAA9GxN3AM8Cdw4spSVqoY4Fn5veAG9rMfwHYMoxQkqTOPJRekgplgUtSoSxwSSqUBS5JhbLAJalQFrgkFcrvA9dA+B3l0vJzC1ySCmWBS1KhLHBJKpRj4Fcwx62lsrkFLkmFssAlqVAWuCQVygKXpEJZ4JJUKAtckgplgUtSoSxwSSqUBS5JhbLAJalQFrgkFcoCl6RCWeCSVCgLXJIKZYFLUqEscEkqlAUuSYWywCWpUBa4JBXKApekQnVd4BGxIiK+FREHqum1EXEwIk5Ul2uGF1OStFCdLfB7gOMt07uBQ5m5EThUTUuSlklXBR4R1wLvBz7dMnsbsK+6vg+4Y6DJJElLiszsvFDEF4A/BV4P/G5mfiAizmXm6pZlzmbmJcMoEbET2AkwPj6+eXp6unbI2dlZxsbGat9v2EaV6+ip80vePn4VnH7p4vSm9atqrWex5XvJ1Lqupr6O0Nxs5qqnqbmgv2xTU1NHMnNi4fxXd7pjRHwAOJOZRyJisu4DZ+YeYA/AxMRETk7WXgUzMzP0cr9hG1WuHbu/tOTtuzZd4P6jF1/akx+arLWexZbvJVPrupr6OkJzs5mrnqbmguFk61jgwK3A7RHxPuC1wBsi4nPA6YhYl5nPR8Q64MxAk0mSltRxDDwz783MazNzA3AX8JXM/DCwH9heLbYdeHhoKSVJl+hnP/D7gHdHxAng3dW0JGmZdDOE8orMnAFmqusvAFsGH0mS1A2PxJSkQlngklQoC1ySCmWBS1KhLHBJKpQFLkmFssAlqVAWuCQVygKXpEJZ4JJUKAtckgpV67tQpA0dvotc0vJxC1ySCmWBS1KhLHBJKpRj4BqJxcbST973/mVOIpXLLXBJKpQFLkmFssAlqVCOgasIjplLl3ILXJIKZYFLUqEscEkqlAUuSYWywCWpUBa4JBXKApekQlngklQoC1ySCmWBS1KhLHBJKpQFLkmF6ljgEfHaiPh6RHw7Ip6IiE9U89dGxMGIOFFdrhl+XEnSvG62wH8C3JaZNwA3Alsj4hZgN3AoMzcCh6ppSdIy6VjgOWe2mnxN9ZPANmBfNX8fcMcwAkqS2ovM7LxQxArgCPCLwKcy8/cj4lxmrm5Z5mxmXjKMEhE7gZ0A4+Pjm6enp2uHnJ2dZWxsrPb9hm1UuY6eOr/k7eNXwemXLk5vWr+q1noWW76bx15qXa3PV93H7iVrHb7H6jFXff1km5qaOpKZEwvnd1XgrywcsRp4CPgt4KvdFHiriYmJPHz4cNePN29mZobJycna9xu2UeVa7OQG83ZtusD9Ry+eq2Oxkx70cpKETo+91Lpan6+6jz3sEzr4HqvHXPX1ky0i2hZ4rb1QMvMcMANsBU5HxLpq5euAMz0lkyT1pJu9UK6ptryJiKuAdwFPAvuB7dVi24GHh5RRktRGN+fEXAfsq8bBXwU8mJkHIuJfgQcj4m7gGeDOIeaU2vJcmbqSdSzwzPwOcFOb+S8AW4YRSpLUmUdiSlKhLHBJKpQFLkmFssAlqVAWuCQVygKXpEJ1sx+4Clf38PdhPfauTRfYMcIssPhzsXfrymVOIvXPLXBJKpQFLkmFssAlqVAWuCQVygKXpEJZ4JJUKAtckgplgUtSoSxwSSqUBS5JhbLAJalQxXwXiuc+XD6j/O4USd1zC1ySCmWBS1KhLHBJKpQFLkmFssAlqVAWuCQVygKXpEIVsx/45cx93CX1wi1wSSqUBS5JhbLAJalQFrgkFapjgUfEmyLisYg4HhFPRMQ91fy1EXEwIk5Ul2uGH1eSNK+bLfALwK7MfAtwC/DRiHgrsBs4lJkbgUPVtCRpmXQs8Mx8PjO/WV3/MXAcWA9sA/ZVi+0D7hhSRklSG7XGwCNiA3AT8DVgPDOfh7mSB9448HSSpEVFZna3YMQY8E/An2TmFyPiXGaubrn9bGZeMg4eETuBnQDj4+Obp6ena4ecnZ3l6fMvt71t0/pVtdc3KLOzs4yNjfW9nqOnzredv9jvttjy88avgtMv9R1r4LrJ1evv3O96rlu1YiCv5aAN6j02aOaqr59sU1NTRzJzYuH8rgo8Il4DHAAezcxPVvOeAiYz8/mIWAfMZOb1S61nYmIiDx8+XDv8zMwMOx55se1tozxacWZmhsnJyb7XU/dIzE5nzNm16QL3H23eQbbd5Or1d+53PXu3rhzIazlog3qPDZq56usnW0S0LfBu9kIJ4DPA8fnyruwHtlfXtwMP95RMktSTbjbTbgU+AhyNiMereX8A3Ac8GBF3A88Adw4loSSprY4FnplfBWKRm7cMNo4kqVseiSlJhbLAJalQFrgkFap5+5pJDbLU7ot1d2H1xB0aNLfAJalQFrgkFcoCl6RCOQauRql7yHwTDep32LD7S+zadIEdC9bnmLnmuQUuSYWywCWpUBa4JBXKApekQlngklQoC1ySCmWBS1KhLHBJKpQFLkmFssAlqVAWuCQV6rL9LhS/e1nS5c4tcEkqlAUuSYWywCWpUJftGPjl4HL4bmxJw+MWuCQVygKXpEJZ4JJUKAtckgplgUtSoSxwSSqUBS5JhXI/cF2W3IdeV4KOW+AR8dmIOBMRx1rmrY2IgxFxorpcM9yYkqSFuhlC2QtsXTBvN3AoMzcCh6ppSdIy6ljgmfnPwH8tmL0N2Fdd3wfcMdhYkqROIjM7LxSxATiQmW+rps9l5uqW289mZtthlIjYCewEGB8f3zw9PV075OzsLE+ff7ntbZvWr2o7/+ip87WW78Xs7CxjY2N9r2exrL0avwpOvzTQVQ5EU3MBXLdqRdvXcqnXpu57r5f1tHvOBvke7tWg3vuD1tRc0F+2qampI5k5sXD+0Au81cTERB4+fLhObgBmZmbY8ciLbW9b7AQNy3FCh5mZGSYnJ/tez6A/cNu16QL3H23e59NNzQWwd+vKtq/lUq9N3fdeL+tp95w14aQkg3rvD1pTc0F/2SKibYH3uhvh6YhYV614HXCmx/VIknrUa4HvB7ZX17cDDw8mjiSpWx3/no2IB4BJ4OqIeBb4Y+A+4MGIuBt4BrhzmCGlJmravuaeB/bK07HAM/ODi9y0ZcBZJEk1eCi9JBXKApekQjVzn66GWWxsce/WlcucRMNy9NR5djRsTFvqxC1wSSqUBS5JhbLAJalQjoG3GPZ+vU3bb1hXhrr7hw9q+aXuo8FwC1ySCmWBS1KhLHBJKlTxY+CjHFdebN9hx/1UR933sJ+laJ5b4JJUKAtckgplgUtSoSxwSSqUBS5JhbLAJalQFrgkFcoCl6RCFX8gj6Tl0XoA0a5NF7o6AYYnWh4ut8AlqVAWuCQVygKXpEJdcWPgy/FFQH7ZkLS0YY+NXylj726BS1KhLHBJKpQFLkmFuuLGwCXNuRw+q1n4O3S7f/rlwi1wSSqUBS5JhbLAJalQjoFLaoxR7b89qMdd6nOFvVtX1lpXN/raAo+IrRHxVER8NyJ2DyqUJKmzngs8IlYAnwLeC7wV+GBEvHVQwSRJS+tnC/xm4LuZ+b3M/B9gGtg2mFiSpE4iM3u7Y8SvA1sz8zer6Y8A78jMjy1Ybiews5q8Hniqh4e7GvhhT0GHy1z1NDUXNDebueppai7oL9svZOY1C2f28yFmtJl3yf8GmbkH2NPH4xARhzNzop91DIO56mlqLmhuNnPV09RcMJxs/QyhPAu8qWX6WuC5/uJIkrrVT4F/A9gYEddFxM8AdwH7BxNLktRJz0MomXkhIj4GPAqsAD6bmU8MLNn/19cQzBCZq56m5oLmZjNXPU3NBUPI1vOHmJKk0fJQekkqlAUuSYVqdIE36VD9iPhsRJyJiGMt89ZGxMGIOFFdrlnmTG+KiMci4nhEPBER9zQhV5XhtRHx9Yj4dpXtE03JVuVYERHfiogDTckVEScj4mhEPB4RhxuUa3VEfCEinqzea+9sSK7rq+dq/udHEfHxhmT7nep9fywiHqj+PQw8V2MLvIGH6u8Fti6Ytxs4lJkbgUPV9HK6AOzKzLcAtwAfrZ6jUecC+AlwW2beANwIbI2IWxqSDeAe4HjLdFNyTWXmjS37Czch118Aj2TmLwE3MPe8jTxXZj5VPVc3ApuB/wYeGnW2iFgP/DYwkZlvY24nj7uGkiszG/kDvBN4tGX6XuDeEWfaABxrmX4KWFddXwc8NeJ8DwPvbmCu1wHfBN7RhGzMHbNwCLgNONCU1xI4CVy9YN5IcwFvAJ6m2uGhKbna5HwP8C9NyAasB74PrGVuT78DVb6B52rsFjgXn4R5z1bzmmQ8M58HqC7fOKogEbEBuAn4WlNyVcMUjwNngIOZ2ZRsfw78HvC/LfOakCuBL0fEkeorKJqQ683AD4C/roacPh0RKxuQa6G7gAeq6yPNlpmngD8DngGeB85n5peHkavJBd7VofqCiBgD/gH4eGb+aNR55mXmyzn35+21wM0R8bYRRyIiPgCcycwjo87Sxq2Z+Xbmhg0/GhG/NupAzG1Bvh34y8y8CXiR0Q0vtVUdSHg78PlRZwGoxra3AdcBPw+sjIgPD+OxmlzgJRyqfzoi1gFUl2eWO0BEvIa58v7bzPxiU3K1ysxzwAxznyGMOtutwO0RcZK5b9C8LSI+14BcZOZz1eUZ5sZyb25ArmeBZ6u/ngC+wFyhjzpXq/cC38zM09X0qLO9C3g6M3+QmT8Fvgj8yjByNbnASzhUfz+wvbq+nbkx6GUTEQF8BjiemZ9sSq4q2zURsbq6fhVzb+onR50tM+/NzGszcwNz76mvZOaHR50rIlZGxOvnrzM3Znps1Lky8z+B70fE9dWsLcC/jTrXAh/k4vAJjD7bM8AtEfG66t/oFuY++B18rlF+8NDFhwHvA/4d+A/gD0ec5QHmxrN+ytxWyd3AzzH3YdiJ6nLtMmf6VeaGlb4DPF79vG/Uuapsvwx8q8p2DPijav7Is7VknOTih5ijfi3fDHy7+nli/v0+6lxVhhuBw9Vr+Y/AmibkqrK9DngBWNUyb+TZgE8wt8FyDPgb4GeHkctD6SWpUE0eQpEkLcECl6RCWeCSVCgLXJIKZYFLUqEscEkqlAUuSYX6PyPWgMaBPsrgAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.Age.hist(bins=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "df99440b",
   "metadata": {},
   "outputs": [],
   "source": [
    "extreme=df.Age.mean()+ 3*df.Age.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "17c936a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def impute_nan(df,variable,median,extreme):\n",
    "    df[variable+\"_end_distribution\"]=df[variable].fillna(extreme)\n",
    "    df[variable].fillna(median,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "3b751fb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "impute_nan(df,'Age',median,extreme)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "d33737f2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Age</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Age_end_distribution</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>38.0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>38.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>26.0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>26.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>35.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>35.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Survived   Age     Fare  Age_end_distribution\n",
       "0         0  22.0   7.2500                  22.0\n",
       "1         1  38.0  71.2833                  38.0\n",
       "2         1  26.0   7.9250                  26.0\n",
       "3         1  35.0  53.1000                  35.0\n",
       "4         0  35.0   8.0500                  35.0"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "547c566d",
   "metadata": {},
   "source": [
    "Arbitrary value imputation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "87e17625",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Age</th>\n",
       "      <th>Fare</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>7.2500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>38.0</td>\n",
       "      <td>71.2833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>26.0</td>\n",
       "      <td>7.9250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>53.1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>8.0500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Survived   Age     Fare\n",
       "0         0  22.0   7.2500\n",
       "1         1  38.0  71.2833\n",
       "2         1  26.0   7.9250\n",
       "3         1  35.0  53.1000\n",
       "4         0  35.0   8.0500"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv('titanic.csv',usecols=['Age','Fare','Survived'])\n",
    "df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "6e8dc279",
   "metadata": {},
   "outputs": [],
   "source": [
    "def impute_nan(df,variable):\n",
    "    df[variable+\"_zero\"]=df[variable].fillna('0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "dc3cea11",
   "metadata": {},
   "outputs": [],
   "source": [
    "impute_nan(df,'Age')  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "143e2671",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Age</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Age_zero</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>38.0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>38.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>26.0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>26.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>35.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>35.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Survived   Age     Fare Age_zero\n",
       "0         0  22.0   7.2500     22.0\n",
       "1         1  38.0  71.2833     38.0\n",
       "2         1  26.0   7.9250     26.0\n",
       "3         1  35.0  53.1000     35.0\n",
       "4         0  35.0   8.0500     35.0"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "070f2635",
   "metadata": {},
   "source": [
    "Frequency Category Imputation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "27e02887",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BsmtQual</th>\n",
       "      <th>FireplaceQu</th>\n",
       "      <th>GarageType</th>\n",
       "      <th>SalePrice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Gd</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>208500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>181500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>223500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TA</td>\n",
       "      <td>Gd</td>\n",
       "      <td>Detchd</td>\n",
       "      <td>140000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>250000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  BsmtQual FireplaceQu GarageType  SalePrice\n",
       "0       Gd         NaN     Attchd     208500\n",
       "1       Gd          TA     Attchd     181500\n",
       "2       Gd          TA     Attchd     223500\n",
       "3       TA          Gd     Detchd     140000\n",
       "4       Gd          TA     Attchd     250000"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv(\"https://raw.githubusercontent.com/sharmaroshan/Advanced-House-Price-Prediction/master/train.csv\",\n",
    "              usecols=['BsmtQual','FireplaceQu','GarageType','SalePrice']\n",
    "              )\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "150060a5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SalePrice      0.000000\n",
       "BsmtQual       0.025342\n",
       "GarageType     0.055479\n",
       "FireplaceQu    0.472603\n",
       "dtype: float64"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().mean().sort_values(ascending=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "87774a52",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BsmtQual\n",
       "Ex    121\n",
       "Fa     35\n",
       "Gd    618\n",
       "TA    649\n",
       "Name: BsmtQual, dtype: int64"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##compute frequency with every category in that column\n",
    "\n",
    "df.groupby(['BsmtQual'])['BsmtQual'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "419ae2d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='BsmtQual'>"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEKCAYAAADpfBXhAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAASZElEQVR4nO3df6zdd13H8eeL7hcCk83dLXWdtJoCdoR146aAKIJTV8HQGVkoyGjMpMYMgz+xJUYjWpyKRI2MpAJaBRwVhFUx6KjMiSErdzDYuq5ZQ+dWW9fLkPBDUlh5+8f5Lpy15/ae3nvODvfj85E038/38/18v+d9Tnpf93O+93y/J1WFJKktT5h0AZKk0TPcJalBhrskNchwl6QGGe6S1KAzJl0AwAUXXFArV66cdBmStKTccccdn6+qqUHbvi3CfeXKlczMzEy6DElaUpL851zbPC0jSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkN+ra4QnUcVm758KRLGMr9N7x00iVIapAzd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDmr2ISaPlRWHS0uLMXZIaZLhLUoOGCvckT03y/iT3JtmX5PlJzk9yS5L7uuV5feO3JjmQZH+Sq8ZXviRpkGFn7n8KfKSqnglcBuwDtgC7q2o1sLtbJ8kaYCNwKbAeuDHJslEXLkma27zhnuRc4IXAOwGq6utV9UVgA7CjG7YDuLprbwBuqqpjVXUQOACsG23ZkqRTGWbm/r3ALPCXST6d5B1JngRcVFVHALrlhd34i4EH+/Y/1PU9RpLNSWaSzMzOzi7qSUiSHmuYcD8DuAJ4e1VdDnyV7hTMHDKgr07qqNpeVdNVNT01NTVUsZKk4QwT7oeAQ1V1e7f+fnph/1CS5QDd8mjf+Ev69l8BHB5NuZKkYcwb7lX138CDSZ7RdV0J3APsAjZ1fZuAm7v2LmBjkrOTrAJWA3tGWrUk6ZSGvUL1F4H3JDkL+Bzws/R+MexMch3wAHANQFXtTbKT3i+AR4Drq+r4yCuXJM1pqHCvqjuB6QGbrpxj/DZg28LLkiQthleoSlKDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktSgocI9yf1J7kpyZ5KZru/8JLckua9bntc3fmuSA0n2J7lqXMVLkgY7nZn7i6tqbVVNd+tbgN1VtRrY3a2TZA2wEbgUWA/cmGTZCGuWJM1jMadlNgA7uvYO4Oq+/puq6lhVHQQOAOsW8TiSpNM0bLgX8C9J7kiyueu7qKqOAHTLC7v+i4EH+/Y91PU9RpLNSWaSzMzOzi6seknSQGcMOe4FVXU4yYXALUnuPcXYDOirkzqqtgPbAaanp0/aLklauKFm7lV1uFseBT5I7zTLQ0mWA3TLo93wQ8AlfbuvAA6PqmBJ0vzmDfckT0rylEfbwI8DdwO7gE3dsE3AzV17F7AxydlJVgGrgT2jLlySNLdhTstcBHwwyaPj31tVH0nySWBnkuuAB4BrAKpqb5KdwD3AI8D1VXV8LNVLkgaaN9yr6nPAZQP6HwaunGOfbcC2RVcnSVoQr1CVpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUFDh3uSZUk+neQfu/Xzk9yS5L5ueV7f2K1JDiTZn+SqcRQuSZrb6czcXw/s61vfAuyuqtXA7m6dJGuAjcClwHrgxiTLRlOuJGkYQ4V7khXAS4F39HVvAHZ07R3A1X39N1XVsao6CBwA1o2kWknSUIaduf8J8Abgm319F1XVEYBueWHXfzHwYN+4Q13fYyTZnGQmyczs7Ozp1i1JOoV5wz3JTwJHq+qOIY+ZAX11UkfV9qqarqrpqampIQ8tSRrGGUOMeQHwsiQvAc4Bzk3ybuChJMur6kiS5cDRbvwh4JK+/VcAh0dZtCTp1OaduVfV1qpaUVUr6f2h9F+r6tXALmBTN2wTcHPX3gVsTHJ2klXAamDPyCuXJM1pmJn7XG4Adia5DngAuAagqvYm2QncAzwCXF9VxxddqSRpaKcV7lV1K3Br134YuHKOcduAbYusTZK0QF6hKkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KD5g33JOck2ZPkM0n2Jvmdrv/8JLckua9bnte3z9YkB5LsT3LVOJ+AJOlkw8zcjwE/UlWXAWuB9UmeB2wBdlfVamB3t06SNcBG4FJgPXBjkmVjqF2SNId5w716vtKtntn9K2ADsKPr3wFc3bU3ADdV1bGqOggcANaNsmhJ0qkNdc49ybIkdwJHgVuq6nbgoqo6AtAtL+yGXww82Lf7oa7vxGNuTjKTZGZ2dnYRT0GSdKKhwr2qjlfVWmAFsC7Js04xPIMOMeCY26tquqqmp6amhipWkjSc0/q0TFV9EbiV3rn0h5IsB+iWR7thh4BL+nZbARxebKGSpOEN82mZqSRP7dpPBH4UuBfYBWzqhm0Cbu7au4CNSc5OsgpYDewZcd2SpFM4Y4gxy4Ed3SdengDsrKp/TPIJYGeS64AHgGsAqmpvkp3APcAjwPVVdXw85UuSBpk33Kvqs8DlA/ofBq6cY59twLZFVydJWhCvUJWkBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQfOGe5JLknwsyb4ke5O8vus/P8ktSe7rluf17bM1yYEk+5NcNc4nIEk62TAz90eAX62q7weeB1yfZA2wBdhdVauB3d063baNwKXAeuDGJMvGUbwkabB5w72qjlTVp7r2l4F9wMXABmBHN2wHcHXX3gDcVFXHquogcABYN+K6JUmncFrn3JOsBC4Hbgcuqqoj0PsFAFzYDbsYeLBvt0Nd34nH2pxkJsnM7OzsAkqXJM1l6HBP8mTgA8AvVdWXTjV0QF+d1FG1vaqmq2p6ampq2DIkSUMYKtyTnEkv2N9TVX/fdT+UZHm3fTlwtOs/BFzSt/sK4PBoypUkDWOYT8sEeCewr6re2rdpF7Cpa28Cbu7r35jk7CSrgNXAntGVLEmazxlDjHkBcC1wV5I7u743AjcAO5NcBzwAXANQVXuT7ATuofdJm+ur6vioC5ckzW3ecK+qjzP4PDrAlXPssw3Ytoi6JEmL4BWqktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoOG+YJsSSO2csuHJ13CUO6/4aWTLkEL5MxdkhpkuEtSgwx3SWrQvOGe5F1Jjia5u6/v/CS3JLmvW57Xt21rkgNJ9ie5alyFS5LmNszM/a+A9Sf0bQF2V9VqYHe3TpI1wEbg0m6fG5MsG1m1kqShzBvuVXUb8IUTujcAO7r2DuDqvv6bqupYVR0EDgDrRlOqJGlYCz3nflFVHQHolhd2/RcDD/aNO9T1nSTJ5iQzSWZmZ2cXWIYkaZBR/0E1A/pq0MCq2l5V01U1PTU1NeIyJOn/t4WG+0NJlgN0y6Nd/yHgkr5xK4DDCy9PkrQQCw33XcCmrr0JuLmvf2OSs5OsAlYDexZXoiTpdM17+4Ekfwu8CLggySHgt4EbgJ1JrgMeAK4BqKq9SXYC9wCPANdX1fEx1S5JmsO84V5Vr5xj05VzjN8GbFtMUZKkxfEKVUlqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkho07/3cJenb2cotH550CUO5/4aXPq6P58xdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGjS3ck6xPsj/JgSRbxvU4kqSTjSXckywD3gb8BLAGeGWSNeN4LEnSycY1c18HHKiqz1XV14GbgA1jeixJ0glSVaM/aPJyYH1V/Vy3fi3w3Kp6Xd+YzcDmbvUZwP6RFzJ6FwCfn3QRDfH1HC1fz9FZKq/l06pqatCGcd1bJgP6HvNbpKq2A9vH9PhjkWSmqqYnXUcrfD1Hy9dzdFp4Lcd1WuYQcEnf+grg8JgeS5J0gnGF+yeB1UlWJTkL2AjsGtNjSZJOMJbTMlX1SJLXAf8MLAPeVVV7x/FYj7MldRppCfD1HC1fz9FZ8q/lWP6gKkmaLK9QlaQGGe6S1CDDXZIaZLgPKcn3JfnNJHdPuhZJmo9fkH0KSZYDrwBeBTwb+H3glRMtaglKcsWptlfVpx6vWlqT5Lqqemff+jLgN6vqdyZY1pKW5ELgnEfXq+qBCZazYH5aZoAkr6UX4iuAnd2/m6tq1UQLW6KSfKxrngNMA5+hdxXzs4Hbq+oHJ1XbUpfkvcBTgeuA7wL+Evi3qvq1Sda1FCV5GfDHwHcDR4GnAfuq6tKJFrZAztwHexvwCeBVVTUDkMTfggtUVS8GSHITsLmq7urWnwUYQotQVa9K8grgLuB/gVdW1X9MuKyl6neB5wEfrarLk7yYJfxO3XPug62gdyfLt3b3pP9d4MwJ19SCZz4a7ABVdTewdnLlLH1JVgOvBz4A3A9cm+Q7JlrU0vWNqnoYeEKSJ1TVx1jC/z+duQ/2kaq6Anh7khX0bp9wNMk+4INV9cbJlrdk7UvyDuDd9G4k92pg32RLWvL+Abi+qnYnCfAr9G7/sSRPJUzYF5M8GbgNeE+So8AjE65pwTznPkCST1fV5QP6n07vba9/rFqAJOcAvwD8UNd1G/D2qjo2uaqWtiTnVtWXTuhbXVX3TaqmpSbJ91TVA0meBHyN3hmNnwG+E3hPN5tfcgz3AZIcAt461/aqmnObTpZkA7Ciqt7Wre8BpujN3t9QVe+fZH1LUZI3VNUfdu1rqurv+ra92XeXw0vyqe6dOkk+UFU/PemaRsFz7oMtA54MPGXAvydPsK6l6g089q6gZwHPAV5Ebyav07exr731hG3rH89CGtD//RPfO7EqRsxz7oMdqao3TbqIhpxVVQ/2rX+8qr4AfKF7K6zTlznag9Z1ajVHe0kz3Afzh2O0zutf6f+6RXqnZ3T6ThVIzQTU4+SyJF+i93P/xK5Nt15Vde7kSls4w32wKyddQGNuT/LaqvqL/s4kPw/smVBNS92pAumcuXfTiapq2aRrGAf/oKqx6y7n/hBwDHj0VgPPAc4Grq6qhyZUmtQsw12PmyQ/wrc+f723qv51kvVILTPcJalBfhRSkhpkuEtSgwx3LXlJjie5M8lnknwqyQ+M4Jhrk7zkhL6rk3w2yb1J7k7y8kUcf6Vf/KJx8qOQasHXqmotQJKr6H2pyg8v8phr6d17/p+6414GvAX4sao6mGQV8NEkB6vqjkU+ljRyztzVmnOB/4HeN2klua2b1d+d5Ie6/q8k+YMkdyT5aJJ1SW5N8rkkL0tyFvAm4BXdvq+gd9/5N1fVQYBu+WbgV7tj3ppkumtfkOT+rr0yyb937yhG8q5CGobhrhY8sQvhe4F30PvSBeh9PeI/d7P6y4A7u/4nAbdW1XOALwO/B/wY8FPAm6rq68BvAe+rqrVV9T56H+E8cYY+A6yZp7aj9Gb7V9D7ysY/W/CzlE6Dp2XUgv7TMs8H/rr7lqdPAu9Kcibwoaq6sxv/deAjXfsu4FhVfSPJXcDKOR4jnHxZ/zC3qTgT+PMka4HjwNOHeULSYjlzV1Oq6hPABcBUVd0GvBD4L+BvkrymG/aN+tYFHt+kd+UsVfVN5p7w7KV3Dr7fFfRm79D7UodHf576L///ZeAheu8cpundEVMaO8NdTUnyTHq3bH44ydOAo909bd5JL4yH9WV6t3h+1FuArUlWdo+zEvgl4I+67ffTu6UCQP+naL6T3l1Gvwlc29UmjZ2nZdSCJya5s2sH2FRVx5O8CPj1JN8AvgK8ZvDuA30M2NId9/er6n1JfgP4hyRn0zt98+Kq2t+NfwuwM8m1QP9tFW4EPpDkmu6YX13A85NOm7cfkBYgyQ3Ac4Gruj/ASt9WDHdJapDn3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KD/g9cPdLxrjX17QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "## you can also plot every category of a column \n",
    "##here you see that TA is the most occuring one\n",
    "df.groupby(['BsmtQual'])['BsmtQual'].count().sort_values(ascending=False).plot.bar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "16e69e94",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "    \n",
    "#Replacing Function\n",
    "def impute_nan(df,variable):\n",
    "    most_frequent_category= df[variable].value_counts().index[0]\n",
    "    df[variable]=df[variable].fillna(most_frequent_category,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "957ca70e",
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "index 0 is out of bounds for axis 0 with size 0",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_13720/2390573112.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mfeature\u001b[0m \u001b[1;32min\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;34m'BsmtQual'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'FireplaceQu'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'GarageType'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m     \u001b[0mimpute_nan\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mfeature\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_13720/4181128418.py\u001b[0m in \u001b[0;36mimpute_nan\u001b[1;34m(df, variable)\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m#Replacing Function\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mimpute_nan\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mvariable\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m     \u001b[0mmost_frequent_category\u001b[0m\u001b[1;33m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mvariable\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalue_counts\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m     \u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mvariable\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mvariable\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfillna\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmost_frequent_category\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0minplace\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\core\\indexes\\base.py\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   4602\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mis_scalar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4603\u001b[0m             \u001b[0mkey\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcom\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcast_scalar_indexer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mwarn_float\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 4604\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mgetitem\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   4605\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4606\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mslice\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mIndexError\u001b[0m: index 0 is out of bounds for axis 0 with size 0"
     ]
    }
   ],
   "source": [
    "\n",
    "for feature in ['BsmtQual','FireplaceQu','GarageType']:\n",
    "    impute_nan(df,feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "bfa7afa2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BsmtQual       1460\n",
       "FireplaceQu    1460\n",
       "GarageType     1460\n",
       "SalePrice         0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a9632e31",
   "metadata": {},
   "source": [
    "Make NAN as a new category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "a7113ae9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BsmtQual</th>\n",
       "      <th>FireplaceQu</th>\n",
       "      <th>GarageType</th>\n",
       "      <th>SalePrice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Gd</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>208500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>181500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>223500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TA</td>\n",
       "      <td>Gd</td>\n",
       "      <td>Detchd</td>\n",
       "      <td>140000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>250000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  BsmtQual FireplaceQu GarageType  SalePrice\n",
       "0       Gd         NaN     Attchd     208500\n",
       "1       Gd          TA     Attchd     181500\n",
       "2       Gd          TA     Attchd     223500\n",
       "3       TA          Gd     Detchd     140000\n",
       "4       Gd          TA     Attchd     250000"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv(\"https://raw.githubusercontent.com/sharmaroshan/Advanced-House-Price-Prediction/master/train.csv\",\n",
    "              usecols=['BsmtQual','FireplaceQu','GarageType','SalePrice']\n",
    "              )\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "9cba473f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def impute_nan(df,variable):\n",
    "    df[variable+\"new_var\"]=np.where(df[variable].isnull(),\"Missing\",df[variable])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "375bd1d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "for feature in ['BsmtQual','FireplaceQu','GarageType']:\n",
    "    impute_nan(df,feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "53ee6688",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BsmtQual</th>\n",
       "      <th>FireplaceQu</th>\n",
       "      <th>GarageType</th>\n",
       "      <th>SalePrice</th>\n",
       "      <th>BsmtQualnew_var</th>\n",
       "      <th>FireplaceQunew_var</th>\n",
       "      <th>GarageTypenew_var</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Gd</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>208500</td>\n",
       "      <td>Gd</td>\n",
       "      <td>Missing</td>\n",
       "      <td>Attchd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>181500</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>223500</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TA</td>\n",
       "      <td>Gd</td>\n",
       "      <td>Detchd</td>\n",
       "      <td>140000</td>\n",
       "      <td>TA</td>\n",
       "      <td>Gd</td>\n",
       "      <td>Detchd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>250000</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  BsmtQual FireplaceQu GarageType  SalePrice BsmtQualnew_var  \\\n",
       "0       Gd         NaN     Attchd     208500              Gd   \n",
       "1       Gd          TA     Attchd     181500              Gd   \n",
       "2       Gd          TA     Attchd     223500              Gd   \n",
       "3       TA          Gd     Detchd     140000              TA   \n",
       "4       Gd          TA     Attchd     250000              Gd   \n",
       "\n",
       "  FireplaceQunew_var GarageTypenew_var  \n",
       "0            Missing            Attchd  \n",
       "1                 TA            Attchd  \n",
       "2                 TA            Attchd  \n",
       "3                 Gd            Detchd  \n",
       "4                 TA            Attchd  "
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c7a564f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80518583",
   "metadata": {},
   "outputs": [],
   "source": [
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a628f1d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "301eb049",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64d45828",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}