Segmentation and Clustering Part1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# #WEEK_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Libraries imported.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # library to handle data in a vectorized manner\n",
    "\n",
    "import pandas as pd # library for data analsysis\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', None)\n",
    "\n",
    "import json # library to handle JSON files\n",
    "\n",
    "#!conda install -c conda-forge geopy --yes # uncomment this line if you haven't completed the Foursquare API lab\n",
    "from geopy.geocoders import Nominatim # convert an address into latitude and longitude values\n",
    "\n",
    "import requests # library to handle requests\n",
    "from pandas.io.json import json_normalize # tranform JSON file into a pandas dataframe\n",
    "\n",
    "# Matplotlib and associated plotting modules\n",
    "import matplotlib.cm as cm\n",
    "import matplotlib.colors as colors\n",
    "\n",
    "#import k-means from clustering stage\n",
    "from sklearn.cluster import KMeans\n",
    "\n",
    "#!conda install -c conda-forge folium=0.5.0 --yes # uncomment this line if you haven't completed the Foursquare API lab\n",
    "import folium # map rendering library\n",
    "\n",
    "#!conda install -c conda-forge beautifulsoup4 --yes\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "print('Libraries imported.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = requests.get('https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M').text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(url, 'lxml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Postcode</th>\n",
       "      <th>Borough</th>\n",
       "      <th>Neighbourhood</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>M1A</td>\n",
       "      <td>Not assigned</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>M2A</td>\n",
       "      <td>Not assigned</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>M3A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Parkwoods</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>M4A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Victoria Village</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>M5A</td>\n",
       "      <td>Downtown Toronto</td>\n",
       "      <td>Regent Park, Harbourfront</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Postcode           Borough              Neighbourhood\n",
       "0      M1A      Not assigned                           \n",
       "1      M2A      Not assigned                           \n",
       "2      M3A        North York                  Parkwoods\n",
       "3      M4A        North York           Victoria Village\n",
       "4      M5A  Downtown Toronto  Regent Park, Harbourfront"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table_post = soup.find('table')\n",
    "td = table_post.find_all('td')\n",
    "\n",
    "postcode = []\n",
    "borough = []\n",
    "neighbourhood = []\n",
    "\n",
    "for i in range(0, len(td), 3):\n",
    "    postcode.append(td[i].text.strip())\n",
    "    borough.append(td[i+1].text.strip())\n",
    "    neighbourhood.append(td[i+2].text.strip())\n",
    "        \n",
    "toronto = pd.DataFrame(data=[postcode, borough, neighbourhood]).transpose()\n",
    "toronto.columns = ['Postcode', 'Borough', 'Neighbourhood']\n",
    "toronto.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Postcode</th>\n",
       "      <th>Borough</th>\n",
       "      <th>Neighbourhood</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>M3A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Parkwoods</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>M4A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Victoria Village</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>M5A</td>\n",
       "      <td>Downtown Toronto</td>\n",
       "      <td>Regent Park, Harbourfront</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>M6A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Lawrence Manor, Lawrence Heights</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>M7A</td>\n",
       "      <td>Downtown Toronto</td>\n",
       "      <td>Queen's Park, Ontario Provincial Government</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Postcode           Borough                                Neighbourhood\n",
       "2      M3A        North York                                    Parkwoods\n",
       "3      M4A        North York                             Victoria Village\n",
       "4      M5A  Downtown Toronto                    Regent Park, Harbourfront\n",
       "5      M6A        North York             Lawrence Manor, Lawrence Heights\n",
       "6      M7A  Downtown Toronto  Queen's Park, Ontario Provincial Government"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "toronto['Borough'].replace('Not assigned', np.nan, inplace=True)\n",
    "toronto.dropna(subset=['Borough'], inplace=True)\n",
    "\n",
    "toronto.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Postcode</th>\n",
       "      <th>Borough</th>\n",
       "      <th>Neighbourhood</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>M1B</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Malvern, Rouge</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>M1C</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Rouge Hill, Port Union, Highland Creek</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>M1E</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Guildwood, Morningside, West Hill</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>M1G</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Woburn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>M1H</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Cedarbrae</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>M1J</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Scarborough Village</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>M1K</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Kennedy Park, Ionview, East Birchmount Park</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>M1L</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Golden Mile, Clairlea, Oakridge</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>M1M</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Cliffside, Cliffcrest, Scarborough Village West</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>M1N</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Birch Cliff, Cliffside West</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>M1P</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Dorset Park, Wexford Heights, Scarborough Town...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>M1R</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Wexford, Maryvale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>M1S</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Agincourt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>M1T</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Clarks Corners, Tam O'Shanter, Sullivan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>M1V</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Milliken, Agincourt North, Steeles East, L'Amo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>M1W</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Steeles West, L'Amoreaux West</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>M1X</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Upper Rouge</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>M2H</td>\n",
       "      <td>North York</td>\n",
       "      <td>Hillcrest Village</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>M2J</td>\n",
       "      <td>North York</td>\n",
       "      <td>Fairview, Henry Farm, Oriole</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>M2K</td>\n",
       "      <td>North York</td>\n",
       "      <td>Bayview Village</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Postcode      Borough                                      Neighbourhood\n",
       "0       M1B  Scarborough                                     Malvern, Rouge\n",
       "1       M1C  Scarborough             Rouge Hill, Port Union, Highland Creek\n",
       "2       M1E  Scarborough                  Guildwood, Morningside, West Hill\n",
       "3       M1G  Scarborough                                             Woburn\n",
       "4       M1H  Scarborough                                          Cedarbrae\n",
       "5       M1J  Scarborough                                Scarborough Village\n",
       "6       M1K  Scarborough        Kennedy Park, Ionview, East Birchmount Park\n",
       "7       M1L  Scarborough                    Golden Mile, Clairlea, Oakridge\n",
       "8       M1M  Scarborough    Cliffside, Cliffcrest, Scarborough Village West\n",
       "9       M1N  Scarborough                        Birch Cliff, Cliffside West\n",
       "10      M1P  Scarborough  Dorset Park, Wexford Heights, Scarborough Town...\n",
       "11      M1R  Scarborough                                  Wexford, Maryvale\n",
       "12      M1S  Scarborough                                          Agincourt\n",
       "13      M1T  Scarborough            Clarks Corners, Tam O'Shanter, Sullivan\n",
       "14      M1V  Scarborough  Milliken, Agincourt North, Steeles East, L'Amo...\n",
       "15      M1W  Scarborough                      Steeles West, L'Amoreaux West\n",
       "16      M1X  Scarborough                                        Upper Rouge\n",
       "17      M2H   North York                                  Hillcrest Village\n",
       "18      M2J   North York                       Fairview, Henry Farm, Oriole\n",
       "19      M2K   North York                                    Bayview Village"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "toronto = toronto.groupby(['Postcode', 'Borough'])['Neighbourhood'].apply(', '.join).reset_index()\n",
    "toronto.columns = ['Postcode', 'Borough', 'Neighbourhood']\n",
    "toronto.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(103, 3)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "toronto.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "toronto.to_csv(r'C:\\Users\\DEVANSH JAIN\\Desktop\\ML specialization\\Coursera_Capstone\\toronto.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}