2.3 Data Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0211fced",
   "metadata": {},
   "source": [
    "## Import Libraries and Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We first would import the necessary libraries we need at the moment, we will import others as when they are needed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3410fac8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd # for data manipulation\n",
    "import joblib # for saving tools"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79ead21d",
   "metadata": {},
   "source": [
    "Import dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9412bf95",
   "metadata": {},
   "outputs": [],
   "source": [
    "lagos = pd.read_csv(\"clean/lagos.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6a8d7e18",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Location</th>\n",
       "      <th>Currency</th>\n",
       "      <th>Price</th>\n",
       "      <th>Serviced</th>\n",
       "      <th>Newly Built</th>\n",
       "      <th>Furnished</th>\n",
       "      <th>Bedrooms</th>\n",
       "      <th>Bathrooms</th>\n",
       "      <th>Toilets</th>\n",
       "      <th>Neighborhood</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Shops</td>\n",
       "      <td>Allen Avenue Roundabout Allen Avenue Ikeja Lagos</td>\n",
       "      <td>₦</td>\n",
       "      <td>750,000/sqm</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0 beds</td>\n",
       "      <td>0 baths</td>\n",
       "      <td>0 Toilets</td>\n",
       "      <td>Allen Avenue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Newly Built 5 Bedrooms Detached Triplex</td>\n",
       "      <td>Eso Close, Off Oduduwa Crescent, Gra Ikeja, La...</td>\n",
       "      <td>₦</td>\n",
       "      <td>280,000,000/year</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>5 beds</td>\n",
       "      <td>5 baths</td>\n",
       "      <td>6 Toilets</td>\n",
       "      <td>GRA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1800m2 Land</td>\n",
       "      <td>Off Adedayo Banjo Street Opebi Ikeja Lagos</td>\n",
       "      <td>₦</td>\n",
       "      <td>220,000,000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0 beds</td>\n",
       "      <td>0 baths</td>\n",
       "      <td>0 Toilets</td>\n",
       "      <td>Opebi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5 Bedroom Fully Detached Duplex</td>\n",
       "      <td>Magodo Ikeja Lagos</td>\n",
       "      <td>₦</td>\n",
       "      <td>300,000,000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>5 beds</td>\n",
       "      <td>6 baths</td>\n",
       "      <td>6 Toilets</td>\n",
       "      <td>Other Ikeja</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Luxury Built 4bedroom Fully Detached Duplex</td>\n",
       "      <td>Ikeja Lagos</td>\n",
       "      <td>₦</td>\n",
       "      <td>295,000,000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4 beds</td>\n",
       "      <td>5 baths</td>\n",
       "      <td>5 Toilets</td>\n",
       "      <td>Other Ikeja</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         Title  \\\n",
       "0                                        Shops   \n",
       "1      Newly Built 5 Bedrooms Detached Triplex   \n",
       "2                                  1800m2 Land   \n",
       "3              5 Bedroom Fully Detached Duplex   \n",
       "4  Luxury Built 4bedroom Fully Detached Duplex   \n",
       "\n",
       "                                            Location Currency  \\\n",
       "0   Allen Avenue Roundabout Allen Avenue Ikeja Lagos        ₦   \n",
       "1  Eso Close, Off Oduduwa Crescent, Gra Ikeja, La...        ₦   \n",
       "2         Off Adedayo Banjo Street Opebi Ikeja Lagos        ₦   \n",
       "3                                 Magodo Ikeja Lagos        ₦   \n",
       "4                                        Ikeja Lagos        ₦   \n",
       "\n",
       "              Price  Serviced  Newly Built  Furnished Bedrooms Bathrooms  \\\n",
       "0       750,000/sqm         1            0          0   0 beds   0 baths   \n",
       "1  280,000,000/year         0            1          0   5 beds   5 baths   \n",
       "2       220,000,000         0            0          0   0 beds   0 baths   \n",
       "3       300,000,000         0            1          0   5 beds   6 baths   \n",
       "4       295,000,000         1            1          0   4 beds   5 baths   \n",
       "\n",
       "     Toilets  Neighborhood  \n",
       "0  0 Toilets  Allen Avenue  \n",
       "1  6 Toilets           GRA  \n",
       "2  0 Toilets         Opebi  \n",
       "3  6 Toilets   Other Ikeja  \n",
       "4  5 Toilets   Other Ikeja  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show first 5 rows\n",
    "lagos.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a520021c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 66383 entries, 0 to 66382\n",
      "Data columns (total 11 columns):\n",
      " #   Column        Non-Null Count  Dtype \n",
      "---  ------        --------------  ----- \n",
      " 0   Title         66382 non-null  object\n",
      " 1   Location      66383 non-null  object\n",
      " 2   Currency      66383 non-null  object\n",
      " 3   Price         66383 non-null  object\n",
      " 4   Serviced      66383 non-null  int64 \n",
      " 5   Newly Built   66383 non-null  int64 \n",
      " 6   Furnished     66383 non-null  int64 \n",
      " 7   Bedrooms      66383 non-null  object\n",
      " 8   Bathrooms     66383 non-null  object\n",
      " 9   Toilets       66383 non-null  object\n",
      " 10  Neighborhood  66383 non-null  object\n",
      "dtypes: int64(3), object(8)\n",
      "memory usage: 5.6+ MB\n"
     ]
    }
   ],
   "source": [
    "# shoe basic information of the dataset\n",
    "lagos.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa989309",
   "metadata": {},
   "source": [
    "We can see from the data above we have 10 features as our Target Variable is **Price**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5d989bbb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Title', 'Location', 'Currency', 'Price', 'Serviced', 'Newly Built',\n",
       "       'Furnished', 'Bedrooms', 'Bathrooms', 'Toilets', 'Neighborhood'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lagos.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8202dc7c",
   "metadata": {},
   "source": [
    "## Feature Engineering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Extracting Type of Property from Title"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d26ec3c",
   "metadata": {},
   "source": [
    "Since we are selling homes, let's parse out the three main types of Property Types from the title column: land, duplex and apartment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "157746bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def house_type(serie):\n",
    "    final = []\n",
    "    \n",
    "    # Extract property name from title\n",
    "    for title in list(serie):\n",
    "        # convert all titles to lowercase trings\n",
    "        title = str(title).lower()\n",
    "        if \"land\" in title and (not \"island\" in title):\n",
    "            final.append(\"land\")\n",
    "        elif (\"duplex\" in title) or (\"massionette\" in title):\n",
    "            final.append(\"duplex\")\n",
    "        else:\n",
    "            final.append(\"apartment\")\n",
    "    return final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8d8f6409",
   "metadata": {},
   "outputs": [],
   "source": [
    "lagos[\"Property Type\"] = house_type(lagos[\"Title\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3f273d11",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Property Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30663</th>\n",
       "      <td>5 Bedroom Detached Duplex With Bq</td>\n",
       "      <td>duplex</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6195</th>\n",
       "      <td>4 Bedroom Terrace House Atisheri North</td>\n",
       "      <td>apartment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58861</th>\n",
       "      <td>4 Bedroom Fully Detached Ikota Villa Estate</td>\n",
       "      <td>apartment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20356</th>\n",
       "      <td>5 Bedroom Semi Detached Duplex With Swimming P...</td>\n",
       "      <td>duplex</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34257</th>\n",
       "      <td>4 Units Of 3 Bedroom Terraced Duplex</td>\n",
       "      <td>duplex</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   Title Property Type\n",
       "30663                  5 Bedroom Detached Duplex With Bq        duplex\n",
       "6195              4 Bedroom Terrace House Atisheri North     apartment\n",
       "58861        4 Bedroom Fully Detached Ikota Villa Estate     apartment\n",
       "20356  5 Bedroom Semi Detached Duplex With Swimming P...        duplex\n",
       "34257               4 Units Of 3 Bedroom Terraced Duplex        duplex"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lagos[[\"Title\",\"Property Type\"]].sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We only need duplex properties, so it will be set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "2c705e4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "lagos = lagos[lagos[\"Property Type\"] == \"duplex\"].copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Extracting Type of Duplex from Title"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc1bd5cc",
   "metadata": {},
   "source": [
    "We can see different types of homes like detached duplex, semi detached etc, lets extract that"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2fb91bcf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4 Bedroom Semi Detached Duplex                                                    1479\n",
       "5 Bedroom Detached Duplex                                                         1088\n",
       "4 Bedroom Terrace Duplex                                                           767\n",
       "5 Bedroom Fully Detached Duplex                                                    761\n",
       "4 Bedroom Detached Duplex                                                          467\n",
       "                                                                                  ... \n",
       "Executive Super Luxury 4 Bedroom Fully Detached Duplex With 1 Room Bq For Sale       1\n",
       "Beautiful 5 Bedroom Fully Detached Duplex In Prime Location                          1\n",
       "Classic 5 Bedroom Fully Detached Duplex                                              1\n",
       "Distinguished 5 Bedroom Contemporary Duplex                                          1\n",
       "7 Bedroom Duplex + 2 Room Bq                                                         1\n",
       "Name: Title, Length: 14667, dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lagos[\"Title\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a7ecaca3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Seperate Detached Duplex Houses from Semi Detached houses\n",
    "lagos.loc[\n",
    "    (lagos[\"Title\"] == \"Detached Duplex\") | (lagos[\"Title\"] == \"5 Bedroom Detached Duplex\") |\n",
    "    (lagos[\"Title\"] == \"4 Bedroom Detached Duplex\") | (lagos[\"Title\"] == \"6 Bedroom Detached Duplex\"),\n",
    "    \"Title\"\n",
    "] = \"ONLY DETACHED DUPLEX\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is a function to differentiate whether a home is detached, semi-detached, terraced or massionette"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "7eb3a80f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def type_house(col):\n",
    "    final = []\n",
    "    \n",
    "    for value in col:\n",
    "        if (\"ONLY DETACHED DUPLEX\" in value.upper()) or ('bedroom duplex' in value.lower()):\n",
    "            final.append(\"Detached Duplex\")\n",
    "        elif ('Fully Detached' in value) or ('bedroom detached duplex' in value.lower()):\n",
    "            final.append(\"Detached Duplex\")\n",
    "        elif ('bedrooms duplex' in value.lower()) or ('bedrooms detached' in value.lower()):\n",
    "            final.append(\"Detached Duplex\")\n",
    "        elif ('bed room' in value.lower()) or ('luxury' in value.lower()) or ('contemporary' in value.lower()):\n",
    "            final.append(\"Detached Duplex\")\n",
    "        elif \"semi\" in value.lower():\n",
    "            final.append(\"Semi Detached Duplex\")\n",
    "        elif \"terra\" in value.lower():\n",
    "            final.append(\"Terraced Duplex\")\n",
    "        else:\n",
    "            final.append(\"Massionette\")\n",
    "            \n",
    "    return final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "15f100be",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Applying the function\n",
    "lagos[\"Type\"] = type_house(lagos[\"Title\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "07758938",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Detached Duplex         19297\n",
       "Semi Detached Duplex     7557\n",
       "Terraced Duplex          5447\n",
       "Massionette               877\n",
       "Name: Type, dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Checking count of type of duplexes\n",
    "lagos[\"Type\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb79a36f",
   "metadata": {},
   "source": [
    "We won't be needing the Title column again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "effb2f40",
   "metadata": {},
   "outputs": [],
   "source": [
    "lagos.drop(\"Title\", axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We dont also need houses that are massionette"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask = lagos.loc[lagos['Type'] == 'Massionette'].index\n",
    "lagos.drop(mask, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cleaning and Casting of the Price column"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9847b70f",
   "metadata": {},
   "source": [
    "It seems our price column is a string, as it contains commas and also includes the price period of the properties\n",
    "\n",
    "Let's sort that out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "af737cc1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30412     90,000,000\n",
       "14242    280,000,000\n",
       "18867     58,000,000\n",
       "33699    500,000,000\n",
       "57243    120,000,000\n",
       "Name: Price, dtype: object"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A sample of price values\n",
    "lagos[\"Price\"].sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Function to differentiate whether a price is fixed or per sqm, per year etc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "bac119ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove the last item from the price\n",
    "def price_split(serie):\n",
    "    final = []\n",
    "    for value in serie:\n",
    "        # if value has a price period, extract it else let it's price period be fixed\n",
    "        try:\n",
    "            final.append(value.split(\"/\")[1])\n",
    "        except:\n",
    "            final.append(\"fixed\")\n",
    "    return final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "1678d185",
   "metadata": {},
   "outputs": [],
   "source": [
    "# applying function\n",
    "lagos[\"Price Period\"] = price_split(lagos[\"Price\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "2d75492e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Price</th>\n",
       "      <th>Price Period</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>29936</th>\n",
       "      <td>180,000,000</td>\n",
       "      <td>fixed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20306</th>\n",
       "      <td>80,000,000</td>\n",
       "      <td>fixed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21169</th>\n",
       "      <td>270,000,000</td>\n",
       "      <td>fixed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30620</th>\n",
       "      <td>58,000,000</td>\n",
       "      <td>fixed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44722</th>\n",
       "      <td>100,000,000</td>\n",
       "      <td>fixed</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Price Price Period\n",
       "29936  180,000,000        fixed\n",
       "20306   80,000,000        fixed\n",
       "21169  270,000,000        fixed\n",
       "30620   58,000,000        fixed\n",
       "44722  100,000,000        fixed"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# checking samples\n",
    "lagos[[\"Price\", \"Price Period\"]].sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02399ade",
   "metadata": {},
   "source": [
    "Now since we have parsed out the price period, let's remove the commas from them and convert them to numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "6a5cca92",
   "metadata": {},
   "outputs": [],
   "source": [
    "def comma_remove(feature):\n",
    "    final = []\n",
    "    for sample in feature:\n",
    "        # Get the prices\n",
    "        sample = sample.split(\"/\")[0]\n",
    "        \n",
    "        # Remove the commas\n",
    "        if \",\" in sample:\n",
    "            final.append(sample.replace(\",\",\"\"))\n",
    "        else:\n",
    "            final.append(sample)\n",
    "    \n",
    "    # Convert them to numbers\n",
    "    int_final = [float(item) for item in final]\n",
    "    return int_final\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "4e4816c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# applying functions\n",
    "lagos[\"Price\"] = comma_remove(lagos[\"Price\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "4d703c74",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Price</th>\n",
       "      <th>Price Period</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11478</th>\n",
       "      <td>1.200000e+09</td>\n",
       "      <td>fixed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35316</th>\n",
       "      <td>1.200000e+08</td>\n",
       "      <td>fixed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31582</th>\n",
       "      <td>1.150000e+08</td>\n",
       "      <td>fixed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44284</th>\n",
       "      <td>2.500000e+08</td>\n",
       "      <td>fixed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31012</th>\n",
       "      <td>1.450000e+08</td>\n",
       "      <td>fixed</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Price Price Period\n",
       "11478  1.200000e+09        fixed\n",
       "35316  1.200000e+08        fixed\n",
       "31582  1.150000e+08        fixed\n",
       "44284  2.500000e+08        fixed\n",
       "31012  1.450000e+08        fixed"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lagos[[\"Price\", \"Price Period\"]].sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extracting houses only with fixed prices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "lagos = lagos[lagos['Price Period'] == 'fixed']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "205123a0",
   "metadata": {},
   "source": [
    "We can now perform numerical methods on the Price column now"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Minimum price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "92aa7af6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lagos[\"Price\"].min()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Average price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "9a7aaaa4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100000000.0"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lagos[\"Price\"].median()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Max Price"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ee5c32f",
   "metadata": {},
   "source": [
    "We can see that some of our other columns meant to be numbers are actually strings, so we would have take care of that"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "b4a4da41",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "140000000000000.0"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lagos[\"Price\"].max()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cleaning and Casting of the Bedrooms, Bathrooms and Toilets Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "2aaf6dbc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Bedrooms</th>\n",
       "      <th>Bathrooms</th>\n",
       "      <th>Toilets</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26649</th>\n",
       "      <td>4 beds</td>\n",
       "      <td>5 baths</td>\n",
       "      <td>5 Toilets</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24980</th>\n",
       "      <td>5 beds</td>\n",
       "      <td>baths</td>\n",
       "      <td>Toilets</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52451</th>\n",
       "      <td>5 beds</td>\n",
       "      <td>5 baths</td>\n",
       "      <td>6 Toilets</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31369</th>\n",
       "      <td>4 beds</td>\n",
       "      <td>4 baths</td>\n",
       "      <td>5 Toilets</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56427</th>\n",
       "      <td>4 beds</td>\n",
       "      <td>4 baths</td>\n",
       "      <td>Toilets</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Bedrooms Bathrooms    Toilets\n",
       "26649   4 beds   5 baths  5 Toilets\n",
       "24980   5 beds     baths    Toilets\n",
       "52451   5 beds   5 baths  6 Toilets\n",
       "31369   4 beds   4 baths  5 Toilets\n",
       "56427   4 beds   4 baths    Toilets"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lagos[[\"Bedrooms\",\"Bathrooms\",\"Toilets\"]].sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Function to clean and cast bathroom, bedroom and toilets features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "e956d01f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_bed_strip(serie):\n",
    "    final = []\n",
    "    # If value doesn't have a number, replace it with a zero,\n",
    "    # else replace it with the number it contains\n",
    "    for value in serie:\n",
    "        if value.split(\"beds\")[0].strip() == \"\":\n",
    "            final.append(int(\"0\"))\n",
    "        else:\n",
    "            final.append(int(value.split(\"beds\")[0].strip()))\n",
    "    return final\n",
    "\n",
    "def remove_toilet_strip(serie):\n",
    "    final = []\n",
    "    for value in serie:\n",
    "        if value.split(\"Toilets\")[0].strip() == \"\":\n",
    "            final.append(int(\"0\"))\n",
    "        else:\n",
    "            final.append(int(value.split(\"Toilets\")[0].strip()))\n",
    "    return final \n",
    "\n",
    "def remove_bath_strip(serie):\n",
    "    final = []\n",
    "    for value in serie:\n",
    "        if value.split(\"baths\")[0].strip() == \"\":\n",
    "            final.append(int(\"0\"))\n",
    "        else:\n",
    "            final.append(int(value.split(\"baths\")[0].strip()))\n",
    "    return final "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Applying functions to three features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "9e009a14",
   "metadata": {},
   "outputs": [],
   "source": [
    "lagos[\"Bedrooms\"] = remove_bed_strip(lagos[\"Bedrooms\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "1b8a8a40",
   "metadata": {},
   "outputs": [],
   "source": [
    "lagos[\"Bathrooms\"] = remove_bath_strip(lagos[\"Bathrooms\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "4f1737a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "lagos[\"Toilets\"] = remove_toilet_strip(lagos[\"Toilets\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "280c3311",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Bedrooms</th>\n",
       "      <th>Bathrooms</th>\n",
       "      <th>Toilets</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9018</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17241</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49419</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12379</th>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1786</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Bedrooms  Bathrooms  Toilets\n",
       "9018          4          4        5\n",
       "17241         0          4        5\n",
       "49419         3          0        0\n",
       "12379         6          6        7\n",
       "1786          4          5        5"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lagos[[\"Bedrooms\",\"Bathrooms\",\"Toilets\"]].sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Extracting City of property"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18a3218e",
   "metadata": {},
   "source": [
    "Also we need to parse out the necessary neighbourhoods from the Location column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "a6c3a7b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove location from data\n",
    "def location_parser(serie):\n",
    "    # Replace all Spaces with a comma in the values\n",
    "    location = [location.replace(\" \", \",\") for location in serie]\n",
    "    \n",
    "    # Split the values by commas, and get the second to the last item, also replace items that\n",
    "    # contain Island with Victoria Island\n",
    "    area = [area.split(\",\")[-2].replace(\"Island\", \"Victoria Island\") for area in location]\n",
    "    \n",
    "    return area"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Applying function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "9e95c476",
   "metadata": {},
   "outputs": [],
   "source": [
    "lagos[\"City\"] = location_parser(lagos[\"Location\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "2baa41e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2883     Ikeja\n",
       "55808    Lekki\n",
       "6633     Ikeja\n",
       "53840    Lekki\n",
       "42399    Lekki\n",
       "Name: City, dtype: object"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lagos[\"City\"].sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We wont be needing the Location feature again since City has been extracted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "011fba64",
   "metadata": {},
   "outputs": [],
   "source": [
    "lagos.drop(\"Location\", axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "dfab78f2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Currency</th>\n",
       "      <th>Price</th>\n",
       "      <th>Serviced</th>\n",
       "      <th>Newly Built</th>\n",
       "      <th>Furnished</th>\n",
       "      <th>Bedrooms</th>\n",
       "      <th>Bathrooms</th>\n",
       "      <th>Toilets</th>\n",
       "      <th>Neighborhood</th>\n",
       "      <th>Property Type</th>\n",
       "      <th>Type</th>\n",
       "      <th>Price Period</th>\n",
       "      <th>City</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>24952</th>\n",
       "      <td>₦</td>\n",
       "      <td>140000000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Ikota</td>\n",
       "      <td>duplex</td>\n",
       "      <td>Detached Duplex</td>\n",
       "      <td>fixed</td>\n",
       "      <td>Lekki</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27719</th>\n",
       "      <td>₦</td>\n",
       "      <td>510000000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>Osapa London</td>\n",
       "      <td>duplex</td>\n",
       "      <td>Detached Duplex</td>\n",
       "      <td>fixed</td>\n",
       "      <td>Lekki</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23592</th>\n",
       "      <td>₦</td>\n",
       "      <td>150000000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>Ikate</td>\n",
       "      <td>duplex</td>\n",
       "      <td>Detached Duplex</td>\n",
       "      <td>fixed</td>\n",
       "      <td>Lekki</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48314</th>\n",
       "      <td>₦</td>\n",
       "      <td>50000000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>Lekki Phase 2</td>\n",
       "      <td>duplex</td>\n",
       "      <td>Semi Detached Duplex</td>\n",
       "      <td>fixed</td>\n",
       "      <td>Lekki</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65743</th>\n",
       "      <td>₦</td>\n",
       "      <td>155000000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Oniru</td>\n",
       "      <td>duplex</td>\n",
       "      <td>Semi Detached Duplex</td>\n",
       "      <td>fixed</td>\n",
       "      <td>Victoria Island</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34070</th>\n",
       "      <td>₦</td>\n",
       "      <td>100000000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>Chevron</td>\n",
       "      <td>duplex</td>\n",
       "      <td>Detached Duplex</td>\n",
       "      <td>fixed</td>\n",
       "      <td>Lekki</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32730</th>\n",
       "      <td>₦</td>\n",
       "      <td>45000000.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>Chevron</td>\n",
       "      <td>duplex</td>\n",
       "      <td>Detached Duplex</td>\n",
       "      <td>fixed</td>\n",
       "      <td>Lekki</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49053</th>\n",
       "      <td>₦</td>\n",
       "      <td>85000000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>Chevron</td>\n",
       "      <td>duplex</td>\n",
       "      <td>Detached Duplex</td>\n",
       "      <td>fixed</td>\n",
       "      <td>Lekki</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43652</th>\n",
       "      <td>₦</td>\n",
       "      <td>125000000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>Chevron</td>\n",
       "      <td>duplex</td>\n",
       "      <td>Detached Duplex</td>\n",
       "      <td>fixed</td>\n",
       "      <td>Lekki</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59836</th>\n",
       "      <td>₦</td>\n",
       "      <td>85000000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>Chevron</td>\n",
       "      <td>duplex</td>\n",
       "      <td>Detached Duplex</td>\n",
       "      <td>fixed</td>\n",
       "      <td>Lekki</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Currency        Price  Serviced  Newly Built  Furnished  Bedrooms  \\\n",
       "24952        ₦  140000000.0         0            1          0         0   \n",
       "27719        ₦  510000000.0         0            1          0         6   \n",
       "23592        ₦  150000000.0         0            1          0         4   \n",
       "48314        ₦   50000000.0         0            1          0         4   \n",
       "65743        ₦  155000000.0         0            0          0         5   \n",
       "34070        ₦  100000000.0         0            1          0         4   \n",
       "32730        ₦   45000000.0         1            1          1         3   \n",
       "49053        ₦   85000000.0         0            1          0         5   \n",
       "43652        ₦  125000000.0         0            1          0         5   \n",
       "59836        ₦   85000000.0         0            0          0         5   \n",
       "\n",
       "       Bathrooms  Toilets   Neighborhood Property Type                  Type  \\\n",
       "24952          0        0          Ikota        duplex       Detached Duplex   \n",
       "27719          6        7   Osapa London        duplex       Detached Duplex   \n",
       "23592          4        5          Ikate        duplex       Detached Duplex   \n",
       "48314          4        5  Lekki Phase 2        duplex  Semi Detached Duplex   \n",
       "65743          0        0          Oniru        duplex  Semi Detached Duplex   \n",
       "34070          4        5        Chevron        duplex       Detached Duplex   \n",
       "32730          4        4        Chevron        duplex       Detached Duplex   \n",
       "49053          5        5        Chevron        duplex       Detached Duplex   \n",
       "43652          5        5        Chevron        duplex       Detached Duplex   \n",
       "59836          5        6        Chevron        duplex       Detached Duplex   \n",
       "\n",
       "      Price Period             City  \n",
       "24952        fixed            Lekki  \n",
       "27719        fixed            Lekki  \n",
       "23592        fixed            Lekki  \n",
       "48314        fixed            Lekki  \n",
       "65743        fixed  Victoria Island  \n",
       "34070        fixed            Lekki  \n",
       "32730        fixed            Lekki  \n",
       "49053        fixed            Lekki  \n",
       "43652        fixed            Lekki  \n",
       "59836        fixed            Lekki  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lagos.sample(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Handling Missing Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After Analysis of the data it was observed that most properties' bathrooms and bedrooms had the same values , while toilets were an increment of the bedrooms. For example a property containing 4 bedrooms will have 4 bathrooms and 5 toilets. Lets turn this idea into a function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "aecfb289",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imputing for bathrooms and bedrooms\n",
    "def bed_bath(cols):\n",
    "    bedroom = cols[0]\n",
    "    bathroom = cols[1]\n",
    "    \n",
    "    if bathroom == 0:\n",
    "\n",
    "        if bedroom == 4:\n",
    "            return 4\n",
    "\n",
    "        elif bedroom == 5:\n",
    "            return 5\n",
    "        \n",
    "        elif bedroom == 6:\n",
    "            return 6\n",
    "\n",
    "        else:\n",
    "            return 0\n",
    "    else:\n",
    "        return bathroom"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "3fe0825e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing for toilets\n",
    "def bed_toilet(cols):\n",
    "    bedroom = cols[0]\n",
    "    toilet = cols[1]\n",
    "    \n",
    "    if toilet == 0:\n",
    "\n",
    "        if bedroom == 4:\n",
    "            return 5\n",
    "\n",
    "        elif bedroom == 5:\n",
    "            return 6\n",
    "\n",
    "        elif bedroom == 6:\n",
    "            return 7\n",
    "        \n",
    "        else:\n",
    "            return 0\n",
    "    else:\n",
    "        return toilet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Applying the functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "d2a2356c",
   "metadata": {},
   "outputs": [],
   "source": [
    "lagos['Bedrooms'] = lagos[['Bathrooms','Bedrooms']].apply(bed_bath,axis=1)\n",
    "lagos['Bathrooms'] = lagos[['Bedrooms','Bathrooms']].apply(bed_bath,axis=1)\n",
    "lagos['Toilets'] = lagos[['Bedrooms','Toilets']].apply(bed_toilet,axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Handling duplicate values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Counting Number of duplicate values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16399"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lagos.duplicated().value_counts()[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Removing duplicate values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "lagos.drop_duplicates(inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Removing Inconsistent rows and features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extracting properties that contain only naira curriencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "12597e75",
   "metadata": {},
   "outputs": [],
   "source": [
    "lagos = lagos.loc[lagos[\"Currency\"] != \"$\"].copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Removing rows that containing only 0 bedrooms, 0 bathrooms and 0 toilets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask = lagos.loc[(lagos['Bedrooms'] == 0) & (lagos['Bedrooms'] == 0) & (lagos['Toilets'] == 0)].index\n",
    "lagos.drop(mask, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Removing Inconsistent Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "1615abc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "lagos.drop(['Currency', 'Property Type', 'Price Period'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Price Cap\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The average Nigerian in 2022 will not normally buy a property less than 10 million naira and higher than 500 million naira, this rows are redundant and will be removed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask = lagos.loc[(lagos[\"Price\"] < 1e7) | (lagos[\"Price\"] > 5e8)].index\n",
    "lagos.drop(mask, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "lagos = lagos.sample(frac=1).copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Saving preprocessed data for EDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "lagos.to_csv('clean/eda.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Data Split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Splitting our data before data preprocessing to avoid data leakage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Spliting to extract 20% test data\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\n",
    "for train_index, test_index in split.split(lagos, lagos[\"Neighborhood\"]):\n",
    "    train_set = lagos.iloc[train_index]\n",
    "    test_set = lagos.iloc[test_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Splitting to extract 20% validation data\n",
    "split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\n",
    "for train_index, test_index in split.split(train_set, train_set[\"Neighborhood\"]):\n",
    "    train = train_set.iloc[train_index]\n",
    "    validation_set = train_set.iloc[test_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "lagos = train.drop(\"Price\", axis=1)\n",
    "price = train[\"Price\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1711dc64",
   "metadata": {},
   "source": [
    "Let's seperate our numerical columns and our categorical ones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "3a80d7e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical_col = [column for column in lagos.columns if lagos[column].dtype == \"O\"]\n",
    "numerical_col = ['Bathrooms', 'Bedrooms', 'Toilets']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "f8abd46b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Neighborhood</th>\n",
       "      <th>Type</th>\n",
       "      <th>City</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>53429</th>\n",
       "      <td>Ikota</td>\n",
       "      <td>Semi Detached Duplex</td>\n",
       "      <td>Lekki</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31650</th>\n",
       "      <td>Ikota</td>\n",
       "      <td>Detached Duplex</td>\n",
       "      <td>Lekki</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56865</th>\n",
       "      <td>Chevron</td>\n",
       "      <td>Semi Detached Duplex</td>\n",
       "      <td>Lekki</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22066</th>\n",
       "      <td>Osapa London</td>\n",
       "      <td>Detached Duplex</td>\n",
       "      <td>Lekki</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Neighborhood                  Type   City\n",
       "53429         Ikota  Semi Detached Duplex  Lekki\n",
       "31650         Ikota       Detached Duplex  Lekki\n",
       "56865       Chevron  Semi Detached Duplex  Lekki\n",
       "22066  Osapa London       Detached Duplex  Lekki"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lagos[categorical_col].sample(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "ec7a353f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Bathrooms</th>\n",
       "      <th>Bedrooms</th>\n",
       "      <th>Toilets</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>48006</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6744</th>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16075</th>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37931</th>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Bathrooms  Bedrooms  Toilets\n",
       "48006          4         4        4\n",
       "6744           5         4        5\n",
       "16075          6         5        6\n",
       "37931          6         6        5"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lagos[numerical_col].sample(4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Handling Missing Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "decb680d",
   "metadata": {},
   "source": [
    "Our dataset does not have missing values, but we still need to set a foundation, in case new data cotains some missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "ff8b5cbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import library for missing data \n",
    "from sklearn.impute import SimpleImputer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a334b5f",
   "metadata": {},
   "source": [
    "Load the imputer and replacing missing categorical variables with their mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "f7c0ba48",
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical_imputer = SimpleImputer(strategy=\"most_frequent\")\n",
    "categorical_imputer.fit(lagos[categorical_col])\n",
    "lagos[categorical_col] = categorical_imputer.transform(lagos[categorical_col])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Saving Imputer for future use in model deployment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['tools/imputer_joblib']"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(categorical_imputer, \"tools/imputer_joblib\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d46148f1",
   "metadata": {},
   "source": [
    "#### Feature Scaling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Scaling for all our features to be in the same range of -3 to 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import library for scaling\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the scaler and scale the values\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(lagos[numerical_col])\n",
    "lagos[numerical_col] = scaler.transform(lagos[numerical_col])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Saving Scaler for future use in model deployment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['tools/scaler_joblib']"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(scaler, \"tools/scaler_joblib\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Feature Encoding"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13e39754",
   "metadata": {},
   "source": [
    "We need to perform OneHotEncoding on some of the norminal categorical variables (variables with no order e.g. Nigeria, Congo, Ghana)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "61510154",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import library for encoding\n",
    "from feature_engine.encoding import OneHotEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = OneHotEncoder()\n",
    "encoder.fit(lagos)\n",
    "lagos = encoder.transform(lagos)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Saving Encoder for future use in model deployment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['tools/encoder_joblib']"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(encoder, \"tools/encoder_joblib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Serviced</th>\n",
       "      <th>Newly Built</th>\n",
       "      <th>Furnished</th>\n",
       "      <th>Bedrooms</th>\n",
       "      <th>Bathrooms</th>\n",
       "      <th>Toilets</th>\n",
       "      <th>Neighborhood_Lekki Phase 1</th>\n",
       "      <th>Neighborhood_Lekki Phase 2</th>\n",
       "      <th>Neighborhood_Ikate</th>\n",
       "      <th>Neighborhood_Ikota</th>\n",
       "      <th>...</th>\n",
       "      <th>Neighborhood_Waziri Adeola Odeku</th>\n",
       "      <th>Neighborhood_Awolowo Way</th>\n",
       "      <th>Neighborhood_Agidingbi</th>\n",
       "      <th>Type_Semi Detached Duplex</th>\n",
       "      <th>Type_Detached Duplex</th>\n",
       "      <th>Type_Terraced Duplex</th>\n",
       "      <th>City_Lekki</th>\n",
       "      <th>City_Ikeja</th>\n",
       "      <th>City_Ikoyi</th>\n",
       "      <th>City_Victoria Island</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>29575</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.483533</td>\n",
       "      <td>-0.568038</td>\n",
       "      <td>-1.045317</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54080</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.483533</td>\n",
       "      <td>-0.568038</td>\n",
       "      <td>-1.045317</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51234</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.765681</td>\n",
       "      <td>-2.449226</td>\n",
       "      <td>0.731838</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44934</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.483533</td>\n",
       "      <td>1.313149</td>\n",
       "      <td>0.731838</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33096</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.765681</td>\n",
       "      <td>-0.568038</td>\n",
       "      <td>-0.156740</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14104</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.765681</td>\n",
       "      <td>0.372556</td>\n",
       "      <td>0.731838</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63029</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.765681</td>\n",
       "      <td>2.253743</td>\n",
       "      <td>1.620415</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27980</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.483533</td>\n",
       "      <td>-0.568038</td>\n",
       "      <td>0.731838</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38099</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.765681</td>\n",
       "      <td>1.313149</td>\n",
       "      <td>0.731838</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52849</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-1.732747</td>\n",
       "      <td>-1.508632</td>\n",
       "      <td>-1.045317</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8430 rows × 45 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Serviced  Newly Built  Furnished  Bedrooms  Bathrooms   Toilets  \\\n",
       "29575         0            0          0 -0.483533  -0.568038 -1.045317   \n",
       "54080         0            0          0 -0.483533  -0.568038 -1.045317   \n",
       "51234         0            0          0  0.765681  -2.449226  0.731838   \n",
       "44934         1            1          1 -0.483533   1.313149  0.731838   \n",
       "33096         0            1          0  0.765681  -0.568038 -0.156740   \n",
       "...         ...          ...        ...       ...        ...       ...   \n",
       "14104         1            1          0  0.765681   0.372556  0.731838   \n",
       "63029         1            0          1  0.765681   2.253743  1.620415   \n",
       "27980         0            1          0 -0.483533  -0.568038  0.731838   \n",
       "38099         0            1          0  0.765681   1.313149  0.731838   \n",
       "52849         1            1          0 -1.732747  -1.508632 -1.045317   \n",
       "\n",
       "       Neighborhood_Lekki Phase 1  Neighborhood_Lekki Phase 2  \\\n",
       "29575                           1                           0   \n",
       "54080                           0                           1   \n",
       "51234                           0                           0   \n",
       "44934                           0                           0   \n",
       "33096                           0                           0   \n",
       "...                           ...                         ...   \n",
       "14104                           0                           0   \n",
       "63029                           0                           0   \n",
       "27980                           1                           0   \n",
       "38099                           0                           0   \n",
       "52849                           0                           0   \n",
       "\n",
       "       Neighborhood_Ikate  Neighborhood_Ikota  ...  \\\n",
       "29575                   0                   0  ...   \n",
       "54080                   0                   0  ...   \n",
       "51234                   1                   0  ...   \n",
       "44934                   0                   1  ...   \n",
       "33096                   0                   0  ...   \n",
       "...                   ...                 ...  ...   \n",
       "14104                   0                   0  ...   \n",
       "63029                   0                   0  ...   \n",
       "27980                   0                   0  ...   \n",
       "38099                   1                   0  ...   \n",
       "52849                   0                   0  ...   \n",
       "\n",
       "       Neighborhood_Waziri Adeola Odeku  Neighborhood_Awolowo Way  \\\n",
       "29575                                 0                         0   \n",
       "54080                                 0                         0   \n",
       "51234                                 0                         0   \n",
       "44934                                 0                         0   \n",
       "33096                                 0                         0   \n",
       "...                                 ...                       ...   \n",
       "14104                                 0                         0   \n",
       "63029                                 0                         0   \n",
       "27980                                 0                         0   \n",
       "38099                                 0                         0   \n",
       "52849                                 0                         0   \n",
       "\n",
       "       Neighborhood_Agidingbi  Type_Semi Detached Duplex  \\\n",
       "29575                       0                          1   \n",
       "54080                       0                          1   \n",
       "51234                       0                          0   \n",
       "44934                       0                          0   \n",
       "33096                       0                          1   \n",
       "...                       ...                        ...   \n",
       "14104                       0                          1   \n",
       "63029                       0                          1   \n",
       "27980                       0                          0   \n",
       "38099                       0                          0   \n",
       "52849                       0                          0   \n",
       "\n",
       "       Type_Detached Duplex  Type_Terraced Duplex  City_Lekki  City_Ikeja  \\\n",
       "29575                     0                     0           1           0   \n",
       "54080                     0                     0           1           0   \n",
       "51234                     1                     0           1           0   \n",
       "44934                     0                     1           1           0   \n",
       "33096                     0                     0           1           0   \n",
       "...                     ...                   ...         ...         ...   \n",
       "14104                     0                     0           0           0   \n",
       "63029                     0                     0           0           0   \n",
       "27980                     0                     1           1           0   \n",
       "38099                     1                     0           1           0   \n",
       "52849                     0                     1           1           0   \n",
       "\n",
       "       City_Ikoyi  City_Victoria Island  \n",
       "29575           0                     0  \n",
       "54080           0                     0  \n",
       "51234           0                     0  \n",
       "44934           0                     0  \n",
       "33096           0                     0  \n",
       "...           ...                   ...  \n",
       "14104           1                     0  \n",
       "63029           0                     1  \n",
       "27980           0                     0  \n",
       "38099           0                     0  \n",
       "52849           0                     0  \n",
       "\n",
       "[8430 rows x 45 columns]"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lagos"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save preprocessed data for Model Building"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "lagos = pd.concat([lagos, price], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "lagos.to_csv('clean/train.csv', index=False)\n",
    "test_set.to_csv('clean/test.csv', index=False)\n",
    "validation_set.to_csv('clean/validation.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.12 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "397fa5c989aefacd265b089cd9fa90a007575229207dba4eead3a1aa550249df"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}