01_zillow_api_avg_prices_by_zip_code.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Zillow API data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import xmltodict\n",
    "import geopandas\n",
    "import geopandas.tools\n",
    "from shapely.geometry import Point\n",
    "import pickle\n",
    "from xml.etree import ElementTree\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Collect & Process Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Set up Zillow API url, and parse the response into a dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "api_key = ''\n",
    "z_url = 'http://www.zillow.com/webservice/GetRegionChildren.htm?zws-id={}&state=c&city=san+francisco&childtype=neighborhood'.format(api_key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "r = requests.get(z_url)\n",
    "response = requests.get(z_url)\n",
    "tree = ElementTree.fromstring(response.content)\n",
    "doc = xmltodict.parse(r.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "z_dict = dict(doc['RegionChildren:regionchildren']['response'])\n",
    "z_dict = z_dict['list']['region']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Extract all the relevant data, store in a list, then drop into a DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "z_data = []\n",
    "for i in range(len(z_dict)):\n",
    "    try:\n",
    "        z_index = int(str(z_dict[i]['zindex']['#text']).replace(\" \", \"\"))\n",
    "    except:\n",
    "        z_index = np.nan\n",
    "    name = str(z_dict[i]['name']).strip()\n",
    "    lat = float(z_dict[i]['latitude'])\n",
    "    lon = float(z_dict[i]['longitude'])\n",
    "    z_data.append([name, z_index, lat, lon])\n",
    "\n",
    "df = pd.DataFrame(z_data)\n",
    "df.columns = ['neighborhood','z_index','latitude','longitude']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Convert the latitude and longitude data into geometry Points, and convert to a GeoDataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighborhood</th>\n",
       "      <th>z_index</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>geometry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Mission</td>\n",
       "      <td>1106500.0</td>\n",
       "      <td>37.759892</td>\n",
       "      <td>-122.415902</td>\n",
       "      <td>POINT (-122.415902 37.759892)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Bernal Heights</td>\n",
       "      <td>1083700.0</td>\n",
       "      <td>37.740357</td>\n",
       "      <td>-122.419694</td>\n",
       "      <td>POINT (-122.419694 37.740357)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Central Richmond</td>\n",
       "      <td>1284800.0</td>\n",
       "      <td>37.778349</td>\n",
       "      <td>-122.482161</td>\n",
       "      <td>POINT (-122.482161 37.778349)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Excelsior</td>\n",
       "      <td>679900.0</td>\n",
       "      <td>37.721890</td>\n",
       "      <td>-122.429767</td>\n",
       "      <td>POINT (-122.429767 37.72189)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Bayview</td>\n",
       "      <td>612400.0</td>\n",
       "      <td>37.729195</td>\n",
       "      <td>-122.391230</td>\n",
       "      <td>POINT (-122.39123 37.729195)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       neighborhood    z_index   latitude   longitude  \\\n",
       "0           Mission  1106500.0  37.759892 -122.415902   \n",
       "1    Bernal Heights  1083700.0  37.740357 -122.419694   \n",
       "2  Central Richmond  1284800.0  37.778349 -122.482161   \n",
       "3         Excelsior   679900.0  37.721890 -122.429767   \n",
       "4           Bayview   612400.0  37.729195 -122.391230   \n",
       "\n",
       "                        geometry  \n",
       "0  POINT (-122.415902 37.759892)  \n",
       "1  POINT (-122.419694 37.740357)  \n",
       "2  POINT (-122.482161 37.778349)  \n",
       "3   POINT (-122.429767 37.72189)  \n",
       "4   POINT (-122.39123 37.729195)  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['geometry'] = df.apply(lambda row: Point(row['longitude'], row['latitude']), axis=1)\n",
    "df = geopandas.GeoDataFrame(df, geometry='geometry')\n",
    "df.crs = {\"init\": \"epsg:4326\"}\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Load in the zip code geospatial data downloaded from the SF open data website"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "zip_df = geopandas.read_file('zip.geojson')\n",
    "zip_df.drop(['id','multigeom','objectid','po_name','st_area_sh','st_length_','state',\n",
    "             'zip','pop10_sqmi','pop2010','sqmi'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Do a spatial join on the two GeoDataFrames so that I have z_index and zip code in the same DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "result = geopandas.tools.sjoin(df, zip_df, how='left')\n",
    "result.drop(['geometry','index_right','latitude','longitude'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "There are multiple neighborhoods for each zip code, so I will take the mean value for each zip code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "result = result.groupby('zip_code').mean().reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now, all that's left is to pickle the DataFrame to be opened later "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "pickle.dump(result, open('z_index_data.pkl', 'wb'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}