Restaurants_yelp_more_features_final.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Engineering\n",
    "\n",
    "In this notebook, we generate additional features that will help us come up with a more predictive model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import necessary libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Read unprocessed data (old and new datasets already merged)\n",
    "df_unprocessed = pd.read_pickle('./yelp_dataset_processed/yelp_df_all_final.pkl')\n",
    "df_reviews = pd.read_json('./yelp_old/yelp_training_set/yelp_training_set_review.json',lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>business_id</th>\n",
       "      <th>categories</th>\n",
       "      <th>categories_new</th>\n",
       "      <th>city</th>\n",
       "      <th>coordinates.latitude</th>\n",
       "      <th>coordinates.longitude</th>\n",
       "      <th>display_phone</th>\n",
       "      <th>found</th>\n",
       "      <th>found2</th>\n",
       "      <th>full_address</th>\n",
       "      <th>...</th>\n",
       "      <th>photos</th>\n",
       "      <th>price</th>\n",
       "      <th>rating</th>\n",
       "      <th>review_count</th>\n",
       "      <th>review_count_new</th>\n",
       "      <th>stars</th>\n",
       "      <th>state</th>\n",
       "      <th>transactions</th>\n",
       "      <th>type</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>PzOqRohWw7F7YEPBz6AubA</td>\n",
       "      <td>[Food, Bagels, Delis, Restaurants]</td>\n",
       "      <td>[{'alias': 'bagels', 'title': 'Bagels'}, {'ali...</td>\n",
       "      <td>Glendale</td>\n",
       "      <td>33.7136</td>\n",
       "      <td>-112.2</td>\n",
       "      <td>(623) 825-0355</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6520 W Happy Valley Rd\\nSte 101\\nGlendale Az, ...</td>\n",
       "      <td>...</td>\n",
       "      <td>[https://s3-media4.fl.yelpcdn.com/bphoto/efZ-8...</td>\n",
       "      <td>$</td>\n",
       "      <td>3.5</td>\n",
       "      <td>14</td>\n",
       "      <td>77</td>\n",
       "      <td>3.5</td>\n",
       "      <td>AZ</td>\n",
       "      <td>[]</td>\n",
       "      <td>business</td>\n",
       "      <td>https://www.yelp.com/biz/hot-bagels-and-deli-g...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>qarobAbxGSHI7ygf1f7a_Q</td>\n",
       "      <td>[Sandwiches, Restaurants]</td>\n",
       "      <td>[{'alias': 'sandwiches', 'title': 'Sandwiches'...</td>\n",
       "      <td>Gilbert</td>\n",
       "      <td>33.3787</td>\n",
       "      <td>-111.813</td>\n",
       "      <td>(480) 632-6453</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>891 E Baseline Rd\\nSuite 102\\nGilbert, AZ 85233</td>\n",
       "      <td>...</td>\n",
       "      <td>[https://s3-media2.fl.yelpcdn.com/bphoto/eTDf8...</td>\n",
       "      <td>$</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>29</td>\n",
       "      <td>3.5</td>\n",
       "      <td>AZ</td>\n",
       "      <td>[pickup]</td>\n",
       "      <td>business</td>\n",
       "      <td>https://www.yelp.com/biz/jersey-mikes-subs-gil...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>JxVGJ9Nly2FFIs_WpJvkug</td>\n",
       "      <td>[Pizza, Restaurants]</td>\n",
       "      <td>[{'alias': 'italian', 'title': 'Italian'}, {'a...</td>\n",
       "      <td>Scottsdale</td>\n",
       "      <td>33.6175</td>\n",
       "      <td>-111.926</td>\n",
       "      <td>(480) 321-8800</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>14418 N Scottsdale Rd\\nSuite 181\\nScottsdale, ...</td>\n",
       "      <td>...</td>\n",
       "      <td>[https://s3-media2.fl.yelpcdn.com/bphoto/1lTHQ...</td>\n",
       "      <td>$$</td>\n",
       "      <td>3.5</td>\n",
       "      <td>55</td>\n",
       "      <td>163</td>\n",
       "      <td>4</td>\n",
       "      <td>AZ</td>\n",
       "      <td>[pickup]</td>\n",
       "      <td>business</td>\n",
       "      <td>https://www.yelp.com/biz/sauce-pizza-and-wine-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Jj7bcQ6NDfKoz4TXwvYfMg</td>\n",
       "      <td>[Burgers, Restaurants]</td>\n",
       "      <td>[{'alias': 'burgers', 'title': 'Burgers'}, {'a...</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>33.567</td>\n",
       "      <td>-112.116</td>\n",
       "      <td>(602) 870-1111</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>8941 N Black Canyon Hwy\\nPhoenix, AZ 85021</td>\n",
       "      <td>...</td>\n",
       "      <td>[https://s3-media4.fl.yelpcdn.com/bphoto/gcmpO...</td>\n",
       "      <td>$$</td>\n",
       "      <td>3.5</td>\n",
       "      <td>23</td>\n",
       "      <td>81</td>\n",
       "      <td>4</td>\n",
       "      <td>AZ</td>\n",
       "      <td>[]</td>\n",
       "      <td>business</td>\n",
       "      <td>https://www.yelp.com/biz/fuddruckers-phoenix-3...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4IAzFJ159GEaIGX1-y6Bmw</td>\n",
       "      <td>[Burgers, Fast Food, Restaurants]</td>\n",
       "      <td>[{'alias': 'hotdogs', 'title': 'Fast Food'}, {...</td>\n",
       "      <td>Scottsdale</td>\n",
       "      <td>33.5819</td>\n",
       "      <td>-111.882</td>\n",
       "      <td>(480) 451-1803</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>9251 E Shea Blvd\\nScottsdale, AZ 85258</td>\n",
       "      <td>...</td>\n",
       "      <td>[https://s3-media4.fl.yelpcdn.com/bphoto/ofHjs...</td>\n",
       "      <td>$</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3</td>\n",
       "      <td>16</td>\n",
       "      <td>2.5</td>\n",
       "      <td>AZ</td>\n",
       "      <td>[]</td>\n",
       "      <td>business</td>\n",
       "      <td>https://www.yelp.com/biz/mcdonalds-scottsdale-...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              business_id                          categories  \\\n",
       "0  PzOqRohWw7F7YEPBz6AubA  [Food, Bagels, Delis, Restaurants]   \n",
       "1  qarobAbxGSHI7ygf1f7a_Q           [Sandwiches, Restaurants]   \n",
       "2  JxVGJ9Nly2FFIs_WpJvkug                [Pizza, Restaurants]   \n",
       "3  Jj7bcQ6NDfKoz4TXwvYfMg              [Burgers, Restaurants]   \n",
       "4  4IAzFJ159GEaIGX1-y6Bmw   [Burgers, Fast Food, Restaurants]   \n",
       "\n",
       "                                      categories_new        city  \\\n",
       "0  [{'alias': 'bagels', 'title': 'Bagels'}, {'ali...    Glendale   \n",
       "1  [{'alias': 'sandwiches', 'title': 'Sandwiches'...     Gilbert   \n",
       "2  [{'alias': 'italian', 'title': 'Italian'}, {'a...  Scottsdale   \n",
       "3  [{'alias': 'burgers', 'title': 'Burgers'}, {'a...     Phoenix   \n",
       "4  [{'alias': 'hotdogs', 'title': 'Fast Food'}, {...  Scottsdale   \n",
       "\n",
       "  coordinates.latitude coordinates.longitude   display_phone found found2  \\\n",
       "0              33.7136                -112.2  (623) 825-0355     1      1   \n",
       "1              33.3787              -111.813  (480) 632-6453     1      1   \n",
       "2              33.6175              -111.926  (480) 321-8800     1      1   \n",
       "3               33.567              -112.116  (602) 870-1111     1      1   \n",
       "4              33.5819              -111.882  (480) 451-1803     1      1   \n",
       "\n",
       "                                        full_address  \\\n",
       "0  6520 W Happy Valley Rd\\nSte 101\\nGlendale Az, ...   \n",
       "1    891 E Baseline Rd\\nSuite 102\\nGilbert, AZ 85233   \n",
       "2  14418 N Scottsdale Rd\\nSuite 181\\nScottsdale, ...   \n",
       "3         8941 N Black Canyon Hwy\\nPhoenix, AZ 85021   \n",
       "4             9251 E Shea Blvd\\nScottsdale, AZ 85258   \n",
       "\n",
       "                         ...                          \\\n",
       "0                        ...                           \n",
       "1                        ...                           \n",
       "2                        ...                           \n",
       "3                        ...                           \n",
       "4                        ...                           \n",
       "\n",
       "                                              photos price rating  \\\n",
       "0  [https://s3-media4.fl.yelpcdn.com/bphoto/efZ-8...     $    3.5   \n",
       "1  [https://s3-media2.fl.yelpcdn.com/bphoto/eTDf8...     $      3   \n",
       "2  [https://s3-media2.fl.yelpcdn.com/bphoto/1lTHQ...    $$    3.5   \n",
       "3  [https://s3-media4.fl.yelpcdn.com/bphoto/gcmpO...    $$    3.5   \n",
       "4  [https://s3-media4.fl.yelpcdn.com/bphoto/ofHjs...     $    2.5   \n",
       "\n",
       "  review_count review_count_new stars state transactions      type  \\\n",
       "0           14               77   3.5    AZ           []  business   \n",
       "1           10               29   3.5    AZ     [pickup]  business   \n",
       "2           55              163     4    AZ     [pickup]  business   \n",
       "3           23               81     4    AZ           []  business   \n",
       "4            3               16   2.5    AZ           []  business   \n",
       "\n",
       "                                                 url  \n",
       "0  https://www.yelp.com/biz/hot-bagels-and-deli-g...  \n",
       "1  https://www.yelp.com/biz/jersey-mikes-subs-gil...  \n",
       "2  https://www.yelp.com/biz/sauce-pizza-and-wine-...  \n",
       "3  https://www.yelp.com/biz/fuddruckers-phoenix-3...  \n",
       "4  https://www.yelp.com/biz/mcdonalds-scottsdale-...  \n",
       "\n",
       "[5 rows x 42 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_unprocessed.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>business_id</th>\n",
       "      <th>date</th>\n",
       "      <th>review_id</th>\n",
       "      <th>stars</th>\n",
       "      <th>text</th>\n",
       "      <th>type</th>\n",
       "      <th>user_id</th>\n",
       "      <th>votes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9yKzy9PApeiPPOUJEtnvkg</td>\n",
       "      <td>2011-01-26</td>\n",
       "      <td>fWKvX83p0-ka4JS3dc6E5A</td>\n",
       "      <td>5</td>\n",
       "      <td>My wife took me here on my birthday for breakf...</td>\n",
       "      <td>review</td>\n",
       "      <td>rLtl8ZkDX5vH5nAx9C3q5Q</td>\n",
       "      <td>{'funny': 0, 'useful': 5, 'cool': 2}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ZRJwVLyzEJq1VAihDhYiow</td>\n",
       "      <td>2011-07-27</td>\n",
       "      <td>IjZ33sJrzXqU-0X6U8NwyA</td>\n",
       "      <td>5</td>\n",
       "      <td>I have no idea why some people give bad review...</td>\n",
       "      <td>review</td>\n",
       "      <td>0a2KyEL0d3Yb1V6aivbIuQ</td>\n",
       "      <td>{'funny': 0, 'useful': 0, 'cool': 0}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6oRAC4uyJCsJl1X0WZpVSA</td>\n",
       "      <td>2012-06-14</td>\n",
       "      <td>IESLBzqUCLdSzSqm0eCSxQ</td>\n",
       "      <td>4</td>\n",
       "      <td>love the gyro plate. Rice is so good and I als...</td>\n",
       "      <td>review</td>\n",
       "      <td>0hT2KtfLiobPvh6cDC8JQg</td>\n",
       "      <td>{'funny': 0, 'useful': 1, 'cool': 0}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>_1QQZuf4zZOyFCvXc0o6Vg</td>\n",
       "      <td>2010-05-27</td>\n",
       "      <td>G-WvGaISbqqaMHlNnByodA</td>\n",
       "      <td>5</td>\n",
       "      <td>Rosie, Dakota, and I LOVE Chaparral Dog Park!!...</td>\n",
       "      <td>review</td>\n",
       "      <td>uZetl9T0NcROGOyFfughhg</td>\n",
       "      <td>{'funny': 0, 'useful': 2, 'cool': 1}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6ozycU1RpktNG2-1BroVtw</td>\n",
       "      <td>2012-01-05</td>\n",
       "      <td>1uJFq2r5QfJG_6ExMRCaGw</td>\n",
       "      <td>5</td>\n",
       "      <td>General Manager Scott Petello is a good egg!!!...</td>\n",
       "      <td>review</td>\n",
       "      <td>vYmM4KTsC8ZfQBg-j5MWkw</td>\n",
       "      <td>{'funny': 0, 'useful': 0, 'cool': 0}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              business_id       date               review_id  stars  \\\n",
       "0  9yKzy9PApeiPPOUJEtnvkg 2011-01-26  fWKvX83p0-ka4JS3dc6E5A      5   \n",
       "1  ZRJwVLyzEJq1VAihDhYiow 2011-07-27  IjZ33sJrzXqU-0X6U8NwyA      5   \n",
       "2  6oRAC4uyJCsJl1X0WZpVSA 2012-06-14  IESLBzqUCLdSzSqm0eCSxQ      4   \n",
       "3  _1QQZuf4zZOyFCvXc0o6Vg 2010-05-27  G-WvGaISbqqaMHlNnByodA      5   \n",
       "4  6ozycU1RpktNG2-1BroVtw 2012-01-05  1uJFq2r5QfJG_6ExMRCaGw      5   \n",
       "\n",
       "                                                text    type  \\\n",
       "0  My wife took me here on my birthday for breakf...  review   \n",
       "1  I have no idea why some people give bad review...  review   \n",
       "2  love the gyro plate. Rice is so good and I als...  review   \n",
       "3  Rosie, Dakota, and I LOVE Chaparral Dog Park!!...  review   \n",
       "4  General Manager Scott Petello is a good egg!!!...  review   \n",
       "\n",
       "                  user_id                                 votes  \n",
       "0  rLtl8ZkDX5vH5nAx9C3q5Q  {'funny': 0, 'useful': 5, 'cool': 2}  \n",
       "1  0a2KyEL0d3Yb1V6aivbIuQ  {'funny': 0, 'useful': 0, 'cool': 0}  \n",
       "2  0hT2KtfLiobPvh6cDC8JQg  {'funny': 0, 'useful': 1, 'cool': 0}  \n",
       "3  uZetl9T0NcROGOyFfughhg  {'funny': 0, 'useful': 2, 'cool': 1}  \n",
       "4  vYmM4KTsC8ZfQBg-j5MWkw  {'funny': 0, 'useful': 0, 'cool': 0}  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_reviews.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's list all the columns to think about our options for feature engineering."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['business_id', 'categories', 'categories_new', 'city',\n",
       "       'coordinates.latitude', 'coordinates.longitude', 'display_phone',\n",
       "       'found', 'found2', 'full_address', 'hours', 'id', 'image_url', 'index',\n",
       "       'is_claimed', 'is_closed', 'latitude', 'location.address1',\n",
       "       'location.address2', 'location.address3', 'location.city',\n",
       "       'location.country', 'location.cross_streets',\n",
       "       'location.display_address', 'location.state', 'location.zip_code',\n",
       "       'longitude', 'name', 'name_new', 'neighborhoods', 'open', 'phone',\n",
       "       'photos', 'price', 'rating', 'review_count', 'review_count_new',\n",
       "       'stars', 'state', 'transactions', 'type', 'url'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_unprocessed.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Each encrypted business_id from the df_unprocessed dataframe has corresponding entries in the df_reviews dataframe with the same business_id."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>business_id</th>\n",
       "      <th>date</th>\n",
       "      <th>review_id</th>\n",
       "      <th>stars</th>\n",
       "      <th>text</th>\n",
       "      <th>type</th>\n",
       "      <th>user_id</th>\n",
       "      <th>votes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>44151</th>\n",
       "      <td>PzOqRohWw7F7YEPBz6AubA</td>\n",
       "      <td>2011-12-05</td>\n",
       "      <td>BNzJxKw6xPVQFDaVIaqIxg</td>\n",
       "      <td>4</td>\n",
       "      <td>FINALLY a decent place for breakfast and lunch...</td>\n",
       "      <td>review</td>\n",
       "      <td>YLE6GoJ61fb0QBF7JTkaJg</td>\n",
       "      <td>{'funny': 5, 'useful': 6, 'cool': 6}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58080</th>\n",
       "      <td>PzOqRohWw7F7YEPBz6AubA</td>\n",
       "      <td>2011-12-08</td>\n",
       "      <td>AeNG9-zB6SeozTFRx6cL-A</td>\n",
       "      <td>4</td>\n",
       "      <td>Stopped in to grab a couple coffees and a bage...</td>\n",
       "      <td>review</td>\n",
       "      <td>A7B8l4bO15tVkpD-5TD2zQ</td>\n",
       "      <td>{'funny': 0, 'useful': 0, 'cool': 2}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67427</th>\n",
       "      <td>PzOqRohWw7F7YEPBz6AubA</td>\n",
       "      <td>2012-07-09</td>\n",
       "      <td>BOpB0T-LbmW2g2D09L7ZGQ</td>\n",
       "      <td>5</td>\n",
       "      <td>Absolutely love this place.  The bagels are ma...</td>\n",
       "      <td>review</td>\n",
       "      <td>s5VnR2vM9zZAqchGQiJtpQ</td>\n",
       "      <td>{'funny': 0, 'useful': 0, 'cool': 0}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82361</th>\n",
       "      <td>PzOqRohWw7F7YEPBz6AubA</td>\n",
       "      <td>2012-12-10</td>\n",
       "      <td>q2UHNFulCcXdATnn7XxnlA</td>\n",
       "      <td>5</td>\n",
       "      <td>You can not go wrong with this place, any bage...</td>\n",
       "      <td>review</td>\n",
       "      <td>31OZmHva7Q0dTNqpM4TKug</td>\n",
       "      <td>{'funny': 0, 'useful': 0, 'cool': 0}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101541</th>\n",
       "      <td>PzOqRohWw7F7YEPBz6AubA</td>\n",
       "      <td>2012-03-21</td>\n",
       "      <td>0VJ8CyG-411PkKOm2Q0BEQ</td>\n",
       "      <td>5</td>\n",
       "      <td>Love this place!  I'm so glad they opened up i...</td>\n",
       "      <td>review</td>\n",
       "      <td>4iyTyF7726rzcg-LhP73-w</td>\n",
       "      <td>{'funny': 0, 'useful': 0, 'cool': 0}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105139</th>\n",
       "      <td>PzOqRohWw7F7YEPBz6AubA</td>\n",
       "      <td>2011-12-20</td>\n",
       "      <td>dT6tQnOc5Rz9ibM6UjDLLw</td>\n",
       "      <td>4</td>\n",
       "      <td>Good stuff...tho will overcharge you on items ...</td>\n",
       "      <td>review</td>\n",
       "      <td>GU6rcPRADsnXvaMFShimhQ</td>\n",
       "      <td>{'funny': 0, 'useful': 0, 'cool': 0}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120885</th>\n",
       "      <td>PzOqRohWw7F7YEPBz6AubA</td>\n",
       "      <td>2012-12-30</td>\n",
       "      <td>GS1ImaW939GCETew_wm8Qw</td>\n",
       "      <td>3</td>\n",
       "      <td>Food was okay. Service was good.</td>\n",
       "      <td>review</td>\n",
       "      <td>skry53Pog0dkLcOVMS2Haw</td>\n",
       "      <td>{'funny': 0, 'useful': 0, 'cool': 0}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122698</th>\n",
       "      <td>PzOqRohWw7F7YEPBz6AubA</td>\n",
       "      <td>2012-09-19</td>\n",
       "      <td>Zlz3mLJ3DIZtmwiiOzz9sg</td>\n",
       "      <td>4</td>\n",
       "      <td>Can't believe that I have not tried this place...</td>\n",
       "      <td>review</td>\n",
       "      <td>HeKxpZpPd_WweAuZSBZ_qg</td>\n",
       "      <td>{'funny': 0, 'useful': 1, 'cool': 0}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130027</th>\n",
       "      <td>PzOqRohWw7F7YEPBz6AubA</td>\n",
       "      <td>2011-12-05</td>\n",
       "      <td>gEOGanMecyDrUA4VR-5SMw</td>\n",
       "      <td>4</td>\n",
       "      <td>Am I back in NJ??? I almost thought so after p...</td>\n",
       "      <td>review</td>\n",
       "      <td>2LdmDvRyrXgFsVL1KoRfRA</td>\n",
       "      <td>{'funny': 0, 'useful': 1, 'cool': 0}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170534</th>\n",
       "      <td>PzOqRohWw7F7YEPBz6AubA</td>\n",
       "      <td>2012-11-28</td>\n",
       "      <td>8t-xdqF9D5AwC3nqBMuXvg</td>\n",
       "      <td>1</td>\n",
       "      <td>Went in one time. Bagels are way too salty!! L...</td>\n",
       "      <td>review</td>\n",
       "      <td>-HKgcpX2B-vAYjBmnT__4Q</td>\n",
       "      <td>{'funny': 0, 'useful': 1, 'cool': 0}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191855</th>\n",
       "      <td>PzOqRohWw7F7YEPBz6AubA</td>\n",
       "      <td>2011-12-05</td>\n",
       "      <td>0MGAhqnyhB7hqn7joTeY5A</td>\n",
       "      <td>3</td>\n",
       "      <td>I stopped in earlier this week just to try a b...</td>\n",
       "      <td>review</td>\n",
       "      <td>p7dvahRPZIWL7T6pFA_0dg</td>\n",
       "      <td>{'funny': 0, 'useful': 1, 'cool': 0}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196338</th>\n",
       "      <td>PzOqRohWw7F7YEPBz6AubA</td>\n",
       "      <td>2012-08-28</td>\n",
       "      <td>GL7ZvvGtCrRkUNltwjzjyQ</td>\n",
       "      <td>1</td>\n",
       "      <td>Ordered Italian hero on a sub to go. Bread was...</td>\n",
       "      <td>review</td>\n",
       "      <td>-HH-r5DncFtjz8s0PYjqrA</td>\n",
       "      <td>{'funny': 1, 'useful': 3, 'cool': 1}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223363</th>\n",
       "      <td>PzOqRohWw7F7YEPBz6AubA</td>\n",
       "      <td>2012-09-15</td>\n",
       "      <td>UCe6MGsZ_uGk3TtxCLGlOw</td>\n",
       "      <td>5</td>\n",
       "      <td>Wow, they are fast on Sunday morning.  They co...</td>\n",
       "      <td>review</td>\n",
       "      <td>E-ZCs-ZmPoann2QEA-Y9vw</td>\n",
       "      <td>{'funny': 0, 'useful': 0, 'cool': 0}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>225845</th>\n",
       "      <td>PzOqRohWw7F7YEPBz6AubA</td>\n",
       "      <td>2012-06-11</td>\n",
       "      <td>hg7rapz_KzAqhoOFYhXVoQ</td>\n",
       "      <td>4</td>\n",
       "      <td>Wow what a find this place was.  \\n\\n-Fresh ba...</td>\n",
       "      <td>review</td>\n",
       "      <td>Iycf9KNRhxvR187Qu2zZHg</td>\n",
       "      <td>{'funny': 0, 'useful': 1, 'cool': 0}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   business_id       date               review_id  stars  \\\n",
       "44151   PzOqRohWw7F7YEPBz6AubA 2011-12-05  BNzJxKw6xPVQFDaVIaqIxg      4   \n",
       "58080   PzOqRohWw7F7YEPBz6AubA 2011-12-08  AeNG9-zB6SeozTFRx6cL-A      4   \n",
       "67427   PzOqRohWw7F7YEPBz6AubA 2012-07-09  BOpB0T-LbmW2g2D09L7ZGQ      5   \n",
       "82361   PzOqRohWw7F7YEPBz6AubA 2012-12-10  q2UHNFulCcXdATnn7XxnlA      5   \n",
       "101541  PzOqRohWw7F7YEPBz6AubA 2012-03-21  0VJ8CyG-411PkKOm2Q0BEQ      5   \n",
       "105139  PzOqRohWw7F7YEPBz6AubA 2011-12-20  dT6tQnOc5Rz9ibM6UjDLLw      4   \n",
       "120885  PzOqRohWw7F7YEPBz6AubA 2012-12-30  GS1ImaW939GCETew_wm8Qw      3   \n",
       "122698  PzOqRohWw7F7YEPBz6AubA 2012-09-19  Zlz3mLJ3DIZtmwiiOzz9sg      4   \n",
       "130027  PzOqRohWw7F7YEPBz6AubA 2011-12-05  gEOGanMecyDrUA4VR-5SMw      4   \n",
       "170534  PzOqRohWw7F7YEPBz6AubA 2012-11-28  8t-xdqF9D5AwC3nqBMuXvg      1   \n",
       "191855  PzOqRohWw7F7YEPBz6AubA 2011-12-05  0MGAhqnyhB7hqn7joTeY5A      3   \n",
       "196338  PzOqRohWw7F7YEPBz6AubA 2012-08-28  GL7ZvvGtCrRkUNltwjzjyQ      1   \n",
       "223363  PzOqRohWw7F7YEPBz6AubA 2012-09-15  UCe6MGsZ_uGk3TtxCLGlOw      5   \n",
       "225845  PzOqRohWw7F7YEPBz6AubA 2012-06-11  hg7rapz_KzAqhoOFYhXVoQ      4   \n",
       "\n",
       "                                                     text    type  \\\n",
       "44151   FINALLY a decent place for breakfast and lunch...  review   \n",
       "58080   Stopped in to grab a couple coffees and a bage...  review   \n",
       "67427   Absolutely love this place.  The bagels are ma...  review   \n",
       "82361   You can not go wrong with this place, any bage...  review   \n",
       "101541  Love this place!  I'm so glad they opened up i...  review   \n",
       "105139  Good stuff...tho will overcharge you on items ...  review   \n",
       "120885                   Food was okay. Service was good.  review   \n",
       "122698  Can't believe that I have not tried this place...  review   \n",
       "130027  Am I back in NJ??? I almost thought so after p...  review   \n",
       "170534  Went in one time. Bagels are way too salty!! L...  review   \n",
       "191855  I stopped in earlier this week just to try a b...  review   \n",
       "196338  Ordered Italian hero on a sub to go. Bread was...  review   \n",
       "223363  Wow, they are fast on Sunday morning.  They co...  review   \n",
       "225845  Wow what a find this place was.  \\n\\n-Fresh ba...  review   \n",
       "\n",
       "                       user_id                                 votes  \n",
       "44151   YLE6GoJ61fb0QBF7JTkaJg  {'funny': 5, 'useful': 6, 'cool': 6}  \n",
       "58080   A7B8l4bO15tVkpD-5TD2zQ  {'funny': 0, 'useful': 0, 'cool': 2}  \n",
       "67427   s5VnR2vM9zZAqchGQiJtpQ  {'funny': 0, 'useful': 0, 'cool': 0}  \n",
       "82361   31OZmHva7Q0dTNqpM4TKug  {'funny': 0, 'useful': 0, 'cool': 0}  \n",
       "101541  4iyTyF7726rzcg-LhP73-w  {'funny': 0, 'useful': 0, 'cool': 0}  \n",
       "105139  GU6rcPRADsnXvaMFShimhQ  {'funny': 0, 'useful': 0, 'cool': 0}  \n",
       "120885  skry53Pog0dkLcOVMS2Haw  {'funny': 0, 'useful': 0, 'cool': 0}  \n",
       "122698  HeKxpZpPd_WweAuZSBZ_qg  {'funny': 0, 'useful': 1, 'cool': 0}  \n",
       "130027  2LdmDvRyrXgFsVL1KoRfRA  {'funny': 0, 'useful': 1, 'cool': 0}  \n",
       "170534  -HKgcpX2B-vAYjBmnT__4Q  {'funny': 0, 'useful': 1, 'cool': 0}  \n",
       "191855  p7dvahRPZIWL7T6pFA_0dg  {'funny': 0, 'useful': 1, 'cool': 0}  \n",
       "196338  -HH-r5DncFtjz8s0PYjqrA  {'funny': 1, 'useful': 3, 'cool': 1}  \n",
       "223363  E-ZCs-ZmPoann2QEA-Y9vw  {'funny': 0, 'useful': 0, 'cool': 0}  \n",
       "225845  Iycf9KNRhxvR187Qu2zZHg  {'funny': 0, 'useful': 1, 'cool': 0}  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reviews of the first entry of the df_unprocessed dataframe\n",
    "df_reviews[df_reviews['business_id'] == df_unprocessed['business_id'][0]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can generate features using the reviews metadata."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n",
      "1200\n",
      "1300\n",
      "1400\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1800\n",
      "1900\n",
      "2000\n",
      "2100\n",
      "2200\n",
      "2300\n",
      "2400\n",
      "2500\n",
      "2600\n",
      "2700\n",
      "2800\n",
      "2900\n",
      "3000\n",
      "3100\n",
      "3200\n",
      "3300\n"
     ]
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "regr = LinearRegression()\n",
    "\n",
    "df_unprocessed['reviews_per_week'] = np.nan\n",
    "df_unprocessed['oldest_review'] = np.nan # expressed in days from today\n",
    "df_unprocessed['std_of_stars'] = np.nan\n",
    "df_unprocessed['median_of_stars'] = np.nan\n",
    "df_unprocessed['stars_linear_coef'] = np.nan\n",
    "df_unprocessed['reactions_per_week'] = np.nan\n",
    "\n",
    "\n",
    "# \n",
    "for i, restaurant in df_unprocessed.iterrows():\n",
    "    reviews = df_reviews[df_reviews['business_id'] == restaurant['business_id']]\n",
    "    # difference between newest and oldest review in weeks\n",
    "    delta_time = (max(reviews['date']) - min(reviews['date'])).days/7.\n",
    "    nreviews = len(reviews)\n",
    "    # proxy for oldness: weeks of oldest review since 2014 (all reviews are older than this)\n",
    "    weeks_since_older_review = (datetime.strptime('Jan 1 2014','%b %d %Y')- min(reviews['date'])).days*7\n",
    "    nreactions = 0\n",
    "    for j, review in reviews.iterrows():\n",
    "        nreactions += review['votes']['funny'] + review['votes']['useful'] + review['votes']['cool']\n",
    "    dates = [d.toordinal() for d in reviews['date']] # convert dates to integer\n",
    "    dates = np.asarray(dates)\n",
    "    # fit linear regression model to find coefficient (do stars increase or decrease?)\n",
    "    regr.fit(dates.reshape(-1,1),reviews['stars'].values.reshape(-1,1)) \n",
    "    \n",
    "    #df_unprocessed.loc[i,'reviews_per_week'] = nreviews / delta_time\n",
    "    \n",
    "    # The following way of calculating frequency is better. Since restaurants with a small number of reviews\n",
    "    # that came at the same period would seem like they get reviews very frequently\n",
    "    df_unprocessed.loc[i,'reviews_per_week'] = nreviews / weeks_since_older_review\n",
    "    df_unprocessed.loc[i,'oldest_review'] = weeks_since_older_review\n",
    "    df_unprocessed.loc[i,'std_of_stars'] = np.std(reviews['stars'])\n",
    "    df_unprocessed.loc[i,'median_of_stars'] = np.median(reviews['stars'])\n",
    "    # same for this, use weeks_since_older review instead of delta_time\n",
    "    df_unprocessed.loc[i,'reactions_per_week'] = nreactions / weeks_since_older_review\n",
    "    df_unprocessed.loc[i,'stars_linear_coef'] = regr.coef_[0][0]\n",
    "    \n",
    "    if i % 100 == 0:\n",
    "        print(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we will make a dictionary of restaurant categories that will help us calculate relative quantities. The keys are the restaurant categories and the values are the indices of the restaurants that belong in each category."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create a dictionary of categories with the index of each corresponding restaurant\n",
    "category_dic = {}\n",
    "for i, restaurant in df_unprocessed.iterrows():\n",
    "    for category in restaurant['categories_new']:\n",
    "        alias = category['alias']\n",
    "        try:\n",
    "            category_dic[alias] += [i]\n",
    "        except:\n",
    "            category_dic[alias] = [i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# make price numeric\n",
    "df_unprocessed = df_unprocessed.replace({'price':{'$':1,'$$':2,'$$$':3,'$$$$':4}})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here, we will calculate restaurant density within 1 mile radius and other relative quantities.  The z\\* quantities are relative to the equivalent quantities of the surrounding restaurants."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/malifier/anaconda/envs/Restaurant_prices3/lib/python3.6/site-packages/ipykernel_launcher.py:62: RuntimeWarning: invalid value encountered in double_scalars\n",
      "/Users/malifier/anaconda/envs/Restaurant_prices3/lib/python3.6/site-packages/ipykernel_launcher.py:63: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n",
      "200\n",
      "300\n",
      "400\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/malifier/anaconda/envs/Restaurant_prices3/lib/python3.6/site-packages/ipykernel_launcher.py:61: RuntimeWarning: Mean of empty slice\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n",
      "1200\n",
      "1300\n",
      "1400\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1800\n",
      "1900\n",
      "2000\n",
      "2100\n",
      "2200\n",
      "2300\n",
      "2400\n",
      "2500\n",
      "2600\n",
      "2700\n",
      "2800\n",
      "2900\n",
      "3000\n",
      "3100\n",
      "3200\n",
      "3300\n"
     ]
    }
   ],
   "source": [
    "# this function returns distance in miles given longitude and latitude of two locations\n",
    "import math\n",
    "def distance(ilat,jlat,ilong,jlong):\n",
    "    R = 6371.e3 # earth radius in m\n",
    "    phi1 = math.radians(ilat)\n",
    "    phi2 = math.radians(jlat)\n",
    "    deltaphi = math.radians(jlat-ilat)\n",
    "    deltalambda = math.radians(jlong-ilong)\n",
    "    a = math.sin(deltaphi/2)**2 + math.cos(phi1)*math.cos(phi2) * math.sin(deltalambda/2)**2\n",
    "    c = 2 * math.atan2(math.sqrt(a), math.sqrt(1-a))\n",
    "    d = R * c / 1609.34 # in miles\n",
    "    return d\n",
    "\n",
    "df_unprocessed['restaurant_density'] = np.nan\n",
    "df_unprocessed['restaurant_similar_density'] = np.nan\n",
    "df_unprocessed['zprice_all'] = np.nan\n",
    "df_unprocessed['zreview_count_all'] = np.nan\n",
    "df_unprocessed['zreview_per_week_all'] = np.nan\n",
    "df_unprocessed['zstar_all'] = np.nan\n",
    "df_unprocessed['is_chain'] = np.nan\n",
    "\n",
    "for i, restaurant in df_unprocessed.iterrows():\n",
    "    price_all_list = []\n",
    "    review_count_all_list = []\n",
    "    review_per_week_all_list = []\n",
    "    star_all_list = []\n",
    "    density_similar_list = []\n",
    "    density_all_list = []\n",
    "    \n",
    "    ilong = restaurant['coordinates.longitude']\n",
    "    ilat = restaurant['coordinates.latitude']\n",
    "    for category in restaurant['categories_new']:\n",
    "        alias = category['alias']\n",
    "        for jrestaurant in category_dic[alias]:\n",
    "            jlong = df_unprocessed.loc[jrestaurant]['coordinates.longitude']\n",
    "            jlat = df_unprocessed.loc[jrestaurant]['coordinates.latitude']\n",
    "            dist = distance(ilat,jlat,ilong,jlong)\n",
    "            # if less than 1 mile (include self)\n",
    "            # certain restaurants will be double counted if they belong to the same categories but\n",
    "            # that's ok as it adds a weight for very similar restaurants\n",
    "            if dist <= 1:\n",
    "                density_similar_list += [jrestaurant]\n",
    "  \n",
    "    df_unprocessed.loc[i,'restaurant_similar_density'] = len(density_similar_list)\n",
    "    \n",
    "    # Loop over all resturants to calculate the \"all\" properties\n",
    "    for j, jrestaurant in df_unprocessed.iterrows():\n",
    "        jlong = jrestaurant['coordinates.longitude']\n",
    "        jlat = jrestaurant['coordinates.latitude']\n",
    "        dist = distance(ilat,jlat,ilong,jlong)\n",
    "        # if less than 1 mile (include self in calculation of statistics)\n",
    "        if dist <= 1:\n",
    "            price_all_list += [jrestaurant['price']]\n",
    "            review_count_all_list += [jrestaurant['review_count']]\n",
    "            review_per_week_all_list += [jrestaurant['reviews_per_week']]\n",
    "            star_all_list += [jrestaurant['stars']]\n",
    "            density_all_list += [j]\n",
    "        \n",
    "        \n",
    "    # price and stars are not divided by standard deviation because it is often equal to 0\n",
    "    df_unprocessed.loc[i,'zprice_all'] = (restaurant['price']-np.nanmean(price_all_list))/4.\n",
    "    df_unprocessed.loc[i,'zreview_count_all'] = (restaurant['review_count']-np.nanmean(review_count_all_list))/np.nanstd(review_count_all_list)\n",
    "    df_unprocessed.loc[i,'zreview_per_week_all'] = (restaurant['reviews_per_week']-np.nanmean(review_per_week_all_list))/np.nanstd(review_per_week_all_list)\n",
    "    df_unprocessed.loc[i,'zstar_all'] = (restaurant['stars']-np.nanmean(star_all_list))/5.\n",
    "        \n",
    "    df_unprocessed.loc[i,'restaurant_density'] = len(density_all_list)\n",
    "    df_unprocessed.loc[i,'restaurant_similar_density'] = len(density_similar_list)\n",
    "    # True if there are more than one\n",
    "    df_unprocessed.loc[i,'is_chain'] = (len(df_unprocessed[df_unprocessed['name_new'] == restaurant['name_new']]) > 1)\n",
    "    \n",
    "    if i % 100 == 0:\n",
    "        print(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check if the new calculation for is_chain worked."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>is_chain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Hot Bagels &amp; Deli</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Jersey Mike's Subs</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Sauce</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Fuddruckers</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>McDonald's</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Peter Piper Pizza</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Panda Garden</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Lenny's Burger Shop</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Five Guys Burgers and Fries</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Sonic Drive-In</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Giant Hamburgers</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Tee Pee Tap Room</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Cold Beers &amp; Cheeseburgers</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>5th Avenue Cafe</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Turk's Fish and Chips</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Port of Subs</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Sushi Time</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Ub's Restaurant &amp; Bar</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Matta's Grill and Cantina</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Del Taco</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Windsor</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Nathans Famous Hotdogs</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Oscar's Pier 83 fish and chips</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>KFC</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Burrito Express</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Tortas La Presa #2</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Tailgate Sports Bar &amp; Grill</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Chipotle</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Boston Market</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Jack in the Box</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3297</th>\n",
       "      <td>Arizona Roadhouse 66</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3298</th>\n",
       "      <td>Subway</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3299</th>\n",
       "      <td>Hungry Howie's Pizza</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3300</th>\n",
       "      <td>La Reina Michoacana</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3301</th>\n",
       "      <td>Hal O. Peno's Mexican Cafe</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3302</th>\n",
       "      <td>Pedritos Mexican Food</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3303</th>\n",
       "      <td>Palm Court Restaurant at the Scottsdale Resort</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3304</th>\n",
       "      <td>Rally's</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3305</th>\n",
       "      <td>Tom's Tavern &amp; 1929 Grill</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3306</th>\n",
       "      <td>Sub Factory</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3307</th>\n",
       "      <td>Olympic Flame</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3308</th>\n",
       "      <td>Native New Yorker</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3309</th>\n",
       "      <td>Louisiana Fried Chicken</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3310</th>\n",
       "      <td>Pete's Fish &amp; Chips</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3311</th>\n",
       "      <td>India Gate</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3312</th>\n",
       "      <td>Sacks Art of Sandwicherie</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3313</th>\n",
       "      <td>Smok'n Joe's Roadhouse</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3314</th>\n",
       "      <td>Popeye's Chicken &amp; Biscuits</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3315</th>\n",
       "      <td>Silver Mine Subs</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3316</th>\n",
       "      <td>Schlotzsky's</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3317</th>\n",
       "      <td>Serrano's Mexican Food Restaurants</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3318</th>\n",
       "      <td>Filiberto's Mexican Food</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3319</th>\n",
       "      <td>Hilberto's Mexican Food</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3320</th>\n",
       "      <td>Paradise Bakery &amp; Cafe</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3321</th>\n",
       "      <td>Hungry Howie's Pizza</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3322</th>\n",
       "      <td>Jack-In-the Box Drive Thru</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3323</th>\n",
       "      <td>Heirloom An American Restaurant</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3324</th>\n",
       "      <td>Fat Freddy's Cafe</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3325</th>\n",
       "      <td>Elephant Bar</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3326</th>\n",
       "      <td>Subway</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3327 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                name is_chain\n",
       "0                                  Hot Bagels & Deli     True\n",
       "1                                 Jersey Mike's Subs     True\n",
       "2                                              Sauce     True\n",
       "3                                        Fuddruckers     True\n",
       "4                                         McDonald's     True\n",
       "5                                  Peter Piper Pizza     True\n",
       "6                                       Panda Garden    False\n",
       "7                                Lenny's Burger Shop     True\n",
       "8                        Five Guys Burgers and Fries     True\n",
       "9                                     Sonic Drive-In     True\n",
       "10                                  Giant Hamburgers    False\n",
       "11                                  Tee Pee Tap Room    False\n",
       "12                        Cold Beers & Cheeseburgers    False\n",
       "13                                   5th Avenue Cafe    False\n",
       "14                             Turk's Fish and Chips    False\n",
       "15                                      Port of Subs     True\n",
       "16                                        Sushi Time    False\n",
       "17                             Ub's Restaurant & Bar    False\n",
       "18                         Matta's Grill and Cantina    False\n",
       "19                                          Del Taco     True\n",
       "20                                           Windsor    False\n",
       "21                            Nathans Famous Hotdogs     True\n",
       "22                    Oscar's Pier 83 fish and chips    False\n",
       "23                                               KFC     True\n",
       "24                                   Burrito Express     True\n",
       "25                                Tortas La Presa #2    False\n",
       "26                       Tailgate Sports Bar & Grill    False\n",
       "27                                          Chipotle     True\n",
       "28                                     Boston Market     True\n",
       "29                                   Jack in the Box     True\n",
       "...                                              ...      ...\n",
       "3297                            Arizona Roadhouse 66    False\n",
       "3298                                          Subway     True\n",
       "3299                            Hungry Howie's Pizza     True\n",
       "3300                             La Reina Michoacana    False\n",
       "3301                      Hal O. Peno's Mexican Cafe    False\n",
       "3302                           Pedritos Mexican Food    False\n",
       "3303  Palm Court Restaurant at the Scottsdale Resort    False\n",
       "3304                                         Rally's     True\n",
       "3305                       Tom's Tavern & 1929 Grill    False\n",
       "3306                                     Sub Factory     True\n",
       "3307                                   Olympic Flame    False\n",
       "3308                               Native New Yorker     True\n",
       "3309                         Louisiana Fried Chicken     True\n",
       "3310                             Pete's Fish & Chips     True\n",
       "3311                                      India Gate    False\n",
       "3312                       Sacks Art of Sandwicherie    False\n",
       "3313                          Smok'n Joe's Roadhouse    False\n",
       "3314                     Popeye's Chicken & Biscuits     True\n",
       "3315                                Silver Mine Subs     True\n",
       "3316                                    Schlotzsky's    False\n",
       "3317              Serrano's Mexican Food Restaurants    False\n",
       "3318                        Filiberto's Mexican Food     True\n",
       "3319                         Hilberto's Mexican Food     True\n",
       "3320                          Paradise Bakery & Cafe     True\n",
       "3321                            Hungry Howie's Pizza     True\n",
       "3322                      Jack-In-the Box Drive Thru     True\n",
       "3323                 Heirloom An American Restaurant    False\n",
       "3324                               Fat Freddy's Cafe    False\n",
       "3325                                    Elephant Bar     True\n",
       "3326                                          Subway     True\n",
       "\n",
       "[3327 rows x 2 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_unprocessed[['name','is_chain']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_unprocessed.to_pickle('./yelp_dataset_processed/yelp_df_more_features_all_final.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "168"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(category_dic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}