00a_Parse HTML.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Parse HTML Files\n",
    "\n",
    "This notebook contains the logic to parse the html files into product features. The output is an hdf5 product file with the category as the key."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from lxml import html\n",
    "import gzip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'/Users/juliusremigio/Documents/DSE/Capstone/scraper'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "html_folder = './html_women'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Reading Julian's files\n",
    "def parse(path):\n",
    "    g = gzip.open(path, 'rb')\n",
    "    for l in g:\n",
    "        yield eval(l)\n",
    "\n",
    "def getDF(path):\n",
    "    i = 0\n",
    "    df = {}\n",
    "    for d in parse(path):\n",
    "        df[i] = d\n",
    "        i += 1\n",
    "    return pd.DataFrame.from_dict(df, orient='index')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Int64Index([0], dtype='int64')\n",
      "(603278, 1)\n"
     ]
    }
   ],
   "source": [
    "# read input file and use as master list to find html files\n",
    "df = pd.read_csv('asin_Women.csv', header=None)\n",
    "print df.columns\n",
    "print df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index([u'asin', u'categories', u'title', u'price', u'salesRank', u'imUrl',\n",
      "       u'brand', u'related', u'description'],\n",
      "      dtype='object')\n",
      "(1503384, 9)\n"
     ]
    }
   ],
   "source": [
    "meta = getDF('meta_Clothing_Shoes_and_Jewelry.json.gz')\n",
    "print meta.columns\n",
    "print meta.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# create new dataframe from meta slice\n",
    "if meta is not None:\n",
    "    c5 = meta[meta.asin.isin(df[0].tolist())].reset_index().copy()\n",
    "    meta = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>asin</th>\n",
       "      <th>categories</th>\n",
       "      <th>title</th>\n",
       "      <th>price</th>\n",
       "      <th>salesRank</th>\n",
       "      <th>imUrl</th>\n",
       "      <th>brand</th>\n",
       "      <th>related</th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>0456844570</td>\n",
       "      <td>[[Clothing, Shoes &amp; Jewelry, Women, Accessorie...</td>\n",
       "      <td>RiZ Women's Beautify Crafted &amp;frac12; Rimmed F...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'Clothing': 1180499}</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/31QZTHxv...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'also_viewed': [u'B008MTRT1O', u'B00BUG47S4'...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>0641674791</td>\n",
       "      <td>[[Clothing, Shoes &amp; Jewelry, Women, Accessorie...</td>\n",
       "      <td>Red Crocodile Embossed Leather Travel Wallet</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/31wW2pPm...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ISBN: 0641674791\\nISBN-13: 9780641674792\\nManu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13</td>\n",
       "      <td>0681358173</td>\n",
       "      <td>[[Clothing, Shoes &amp; Jewelry, Women, Petite, Fa...</td>\n",
       "      <td>Bear Ear Hoodie Jacket &amp;amp; Poms on drawstrin...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'Toys &amp; Games': 600198}</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/51JCLkQk...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Adorable fleece teddy bear ear sweater for adu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>19</td>\n",
       "      <td>1028907516</td>\n",
       "      <td>[[Clothing, Shoes &amp; Jewelry, Women, Watches, W...</td>\n",
       "      <td>New Lava Iron Blue Samurai Metal LED Faceless ...</td>\n",
       "      <td>6.29</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/51-qMpRX...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'buy_after_viewing': [u'B00860FTNY', u'B008H...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>24</td>\n",
       "      <td>1304743594</td>\n",
       "      <td>[[Clothing, Shoes &amp; Jewelry, Women, Watches, W...</td>\n",
       "      <td>Watch Black Match Iphone</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/4135gTwj...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{u'also_viewed': [u'1300827327', u'1304743128'...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index        asin                                         categories  \\\n",
       "0      3  0456844570  [[Clothing, Shoes & Jewelry, Women, Accessorie...   \n",
       "1     10  0641674791  [[Clothing, Shoes & Jewelry, Women, Accessorie...   \n",
       "2     13  0681358173  [[Clothing, Shoes & Jewelry, Women, Petite, Fa...   \n",
       "3     19  1028907516  [[Clothing, Shoes & Jewelry, Women, Watches, W...   \n",
       "4     24  1304743594  [[Clothing, Shoes & Jewelry, Women, Watches, W...   \n",
       "\n",
       "                                               title  price  \\\n",
       "0  RiZ Women's Beautify Crafted &frac12; Rimmed F...    NaN   \n",
       "1       Red Crocodile Embossed Leather Travel Wallet    NaN   \n",
       "2  Bear Ear Hoodie Jacket &amp; Poms on drawstrin...    NaN   \n",
       "3  New Lava Iron Blue Samurai Metal LED Faceless ...   6.29   \n",
       "4                           Watch Black Match Iphone    NaN   \n",
       "\n",
       "                   salesRank  \\\n",
       "0     {u'Clothing': 1180499}   \n",
       "1                        NaN   \n",
       "2  {u'Toys & Games': 600198}   \n",
       "3                        NaN   \n",
       "4                        NaN   \n",
       "\n",
       "                                               imUrl brand  \\\n",
       "0  http://ecx.images-amazon.com/images/I/31QZTHxv...   NaN   \n",
       "1  http://ecx.images-amazon.com/images/I/31wW2pPm...   NaN   \n",
       "2  http://ecx.images-amazon.com/images/I/51JCLkQk...   NaN   \n",
       "3  http://ecx.images-amazon.com/images/I/51-qMpRX...   NaN   \n",
       "4  http://ecx.images-amazon.com/images/I/4135gTwj...   NaN   \n",
       "\n",
       "                                             related  \\\n",
       "0  {u'also_viewed': [u'B008MTRT1O', u'B00BUG47S4'...   \n",
       "1                                                NaN   \n",
       "2                                                NaN   \n",
       "3  {u'buy_after_viewing': [u'B00860FTNY', u'B008H...   \n",
       "4  {u'also_viewed': [u'1300827327', u'1304743128'...   \n",
       "\n",
       "                                         description  \n",
       "0                                                NaN  \n",
       "1  ISBN: 0641674791\\nISBN-13: 9780641674792\\nManu...  \n",
       "2  Adorable fleece teddy bear ear sweater for adu...  \n",
       "3                                                NaN  \n",
       "4                                                NaN  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c5.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get Missing Features From HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "index            int64\n",
      "asin            object\n",
      "categories      object\n",
      "title           object\n",
      "price          float64\n",
      "salesRank       object\n",
      "imUrl           object\n",
      "brand           object\n",
      "related         object\n",
      "description     object\n",
      "dtype: object\n",
      "\n",
      "\n",
      "length: 603277\n",
      "missing price: 349160\n",
      "missing brand: 579362\n",
      "missing description 587975\n"
     ]
    }
   ],
   "source": [
    "# count of missing price/brand\n",
    "print c5.dtypes\n",
    "print '\\n'\n",
    "print 'length:', c5.shape[0]\n",
    "print 'missing price:', c5[(c5.price.isnull()) | (c5.price == -1)].shape[0]\n",
    "print 'missing brand:', c5[(c5.brand == '') | (c5.brand.isnull()) | (c5.brand == 'Unknown')].shape[0]\n",
    "print 'missing description', c5[(c5.description == '') | (c5.description.isnull())].shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def extract(asin):\n",
    "    def getDicFromTable(html):\n",
    "        # extract only html nodes from raw html\n",
    "        rows = [x for x in html if str(type(x)) == \"<class 'lxml.html.HtmlElement'>\"]\n",
    "\n",
    "        # return dictionary from list of tuples of headers and data elements\n",
    "        return dict(\n",
    "            zip([x.text.strip() for x in rows if x.tag == 'th'], [x.text.strip() for x in rows if x.tag == 'td']))\n",
    "\n",
    "    def getProductInfo(raw):\n",
    "        # parse tabular list and return dictionary\n",
    "        rows = [x for x in raw if str(type(x)) == \"<class 'lxml.html.HtmlElement'>\"]\n",
    "\n",
    "        mydic = dict()\n",
    "        mylist = [x for x in rows if x.tag == 'li']\n",
    "\n",
    "        for li in mylist:\n",
    "            if len(li.getchildren()) == 1:\n",
    "                mydic[li[0].text.strip()] = li[0].tail.strip() if li[0].tail is not None else ''\n",
    "            else:\n",
    "                mydic[li[0].text.strip()] = li[0].tail.strip() + ''.join([x.text_content().strip() + (x.tail.strip() if x.tail is not None else '') for x in li[1:] if x.tag not in ['ul','style','script']])\n",
    "        return mydic\n",
    "    \n",
    "    source =  u''.join([x.decode('latin-1') for x in open( (html_folder + '/{asin}.html').format(asin=asin), 'r')])\n",
    "    doc = html.fromstring(source)\n",
    "\n",
    "    XPATH_NAME = '//h1[@id=\"title\"]//text()'\n",
    "    XPATH_SALE_PRICE = '//span[contains(@id,\"ourprice\") or contains(@id,\"saleprice\")]/text()'\n",
    "    XPATH_ORIGINAL_PRICE = '//td[contains(text(),\"List Price\") or contains(text(),\"M.R.P\") or contains(text(),\"Price\")]/following-sibling::td/text()'\n",
    "    XPATH_BRAND = '//a[@id=\"brand\"]//text()'\n",
    "    XPATH_BRAND_img = '//a[@id=\"brand\"]/@href'\n",
    "    XPATH_FEATURE_BULLETS = '//div[@id=\"feature-bullets\"]//li/span[@class=\"a-list-item\"]/text()'\n",
    "    XPATH_PRODUCT_INFORMATION = '//table[@id=\"productDetails_detailBullets_sections1\"]//tr/node()'\n",
    "    XPATH_PRODUCT_INFO_li = '//div[@id=\"detailBullets_feature_div\"]//li//text()'\n",
    "    XPATH_PRODUCT_INFO_div2 = '//div[@id=\"detail-bullets\"]//div[@class=\"content\"]/ul/node()'\n",
    "    XPATH_PRODUCT_DESCRIPTION = '//div[@id=\"productDescription\"]//text()'\n",
    "    XPATH_PRODUCT_DESC2 = '//div[@id=\"productDescription\"]//text()'\n",
    "    \n",
    "    RAW_NAME = doc.xpath(XPATH_NAME)\n",
    "    RAW_SALE_PRICE = doc.xpath(XPATH_SALE_PRICE)\n",
    "    RAW_ORIGINAL_PRICE = doc.xpath(XPATH_ORIGINAL_PRICE)\n",
    "    RAW_BRAND = doc.xpath(XPATH_BRAND)\n",
    "    RAW_BRAND_img = doc.xpath(XPATH_BRAND_img)\n",
    "    RAW_FEATURE_BULLETS = doc.xpath(XPATH_FEATURE_BULLETS)\n",
    "    RAW_PRODUCT_INFORMATION = doc.xpath(XPATH_PRODUCT_INFORMATION)\n",
    "    RAW_PRODUCT_INFO_li = doc.xpath(XPATH_PRODUCT_INFO_li)\n",
    "    RAW_PRODUCT_INFO_div2 = doc.xpath(XPATH_PRODUCT_INFO_div2)\n",
    "    RAW_PRODUCT_DESCRIPTION = doc.xpath(XPATH_PRODUCT_DESCRIPTION)\n",
    "    RAW_PRODUCT_DESC2 = doc.xpath(XPATH_PRODUCT_DESC2)\n",
    "    \n",
    "    NAME = ' '.join(''.join(RAW_NAME).split()) if RAW_NAME else None\n",
    "    SALE_PRICE = ' '.join(''.join(RAW_SALE_PRICE).split()).strip() if RAW_SALE_PRICE else None\n",
    "    ORIGINAL_PRICE = ''.join(RAW_ORIGINAL_PRICE).strip() if RAW_ORIGINAL_PRICE else None\n",
    "    BRAND = ''.join(RAW_BRAND).strip()\n",
    "    BRAND_img = None\n",
    "    if len(RAW_BRAND_img) > 0:\n",
    "        BRAND_img = ' '.join(RAW_BRAND_img[0].split('=')[-1].split('+'))\n",
    "    \n",
    "    FEATURE_BULLETS = [x.strip() for x in RAW_FEATURE_BULLETS if x.strip() != '']\n",
    "    PRODUCT_INFORMATION = getDicFromTable(RAW_PRODUCT_INFORMATION)\n",
    "    PRODUCT_INFO_li = [x.strip() for x in RAW_PRODUCT_INFO_li if x is not None and x.strip() != '']\n",
    "    PRODUCT_INFO_div2 = getProductInfo(RAW_PRODUCT_INFO_div2)\n",
    "    PRODUCT_DESCRIPTION = '\\n'.join([x.strip() for x in RAW_PRODUCT_DESCRIPTION if x.strip() != ''])\n",
    "    PRODUCT_DESC2 = ' '.join([x.strip() for x in RAW_PRODUCT_DESC2 if x.strip() != ''])\n",
    "\n",
    "    if not ORIGINAL_PRICE:\n",
    "        ORIGINAL_PRICE = SALE_PRICE\n",
    "    \n",
    "    if PRODUCT_INFORMATION == False:\n",
    "        PRODUCT_INFORMATION = dict(zip(PRODUCT_INFO_li[::2],PRODUCT_INFO_li[1::2])) if len(PRODUCT_INFO_li) > 0 else PRODUCT_INFO_div2\n",
    "       \n",
    "    \n",
    "    return {\n",
    "            'asin' : asin,\n",
    "            'NAME' : NAME,\n",
    "            'SALE_PRICE': SALE_PRICE,\n",
    "            'ORIGINAL_PRICE' : ORIGINAL_PRICE,\n",
    "            'BRAND' : BRAND_img if BRAND is None or BRAND == '' else BRAND,\n",
    "            'PRODUCT_INFORMATION' : PRODUCT_INFORMATION,\n",
    "            'FEATURE_BULLETS' : FEATURE_BULLETS,\n",
    "            'PRODUCT_DESCRIPTION' : PRODUCT_DESC2 if PRODUCT_DESCRIPTION == '' or PRODUCT_DESCRIPTION is None else PRODUCT_DESCRIPTION\n",
    "           }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# extract features from html files\n",
    "files = [x for x in os.listdir(html_folder) if x.endswith('.html')]\n",
    "scraped = [extract(x[:-5]) for x in files]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# merge existing and scraped features\n",
    "combo = pd.merge(c5, pd.DataFrame.from_records(scraped), on='asin', how='left').drop('index', axis='columns')\n",
    "combo.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# save to file\n",
    "combo.to_hdf('womens_Meta_scraped.hd5', key='data', compression='blosc')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}